wehub-resource-sync
874 lines
41 KiB
Python
874 lines
41 KiB
Python
try: from bs4 import BeautifulSoup
|
|
except ImportError: print("[Error] BeautifulSoup4 未安装，请叫Agent安装BeautifulSoup4，再使用web相关工具。")
|
|
|
|
js_optHTML = r'''function optHTML(text_only=false) {
|
|
function createEnhancedDOMCopy() {
|
|
const nodeInfo = new WeakMap();
|
|
const ignoreTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'META', 'LINK', 'COLGROUP', 'COL', 'TEMPLATE', 'PARAM', 'SOURCE'];
|
|
const ignoreIds = ['ljq-ind'];
|
|
function cloneNode(sourceNode, keep=false) {
|
|
if (sourceNode.nodeType === 8 ||
|
|
(sourceNode.nodeType === 1 && (
|
|
ignoreTags.includes(sourceNode.tagName) ||
|
|
(sourceNode.id && ignoreIds.includes(sourceNode.id))
|
|
))) {
|
|
return null;
|
|
}
|
|
if (sourceNode.nodeType === 3) return sourceNode.cloneNode(false);
|
|
const clone = sourceNode.cloneNode(false);
|
|
if ((sourceNode.tagName === 'INPUT' || sourceNode.tagName === 'TEXTAREA') && sourceNode.value) clone.setAttribute('value', sourceNode.value);
|
|
if (sourceNode.tagName === 'INPUT' && (sourceNode.type === 'radio' || sourceNode.type === 'checkbox') && sourceNode.checked) clone.setAttribute('checked', '');
|
|
else if (sourceNode.tagName === 'SELECT' && sourceNode.value) clone.setAttribute('data-selected', sourceNode.value);
|
|
try { if (sourceNode.matches && sourceNode.matches(':-webkit-autofill')) { clone.setAttribute('data-autofilled', 'true'); if (!sourceNode.value) clone.setAttribute('value', '⚠️受保护-读tmwebdriver_sop的autofill章节提取'); } } catch(e) {}
|
|
|
|
const isDropdown = sourceNode.classList?.contains('dropdown-menu') ||
|
|
/dropdown|menu/i.test(sourceNode.className) || sourceNode.getAttribute('role') === 'menu';
|
|
const _ddItems = isDropdown ? sourceNode.querySelectorAll('a, button, [role="menuitem"], li').length : 0;
|
|
const isSmallDropdown = _ddItems > 0 && _ddItems <= 7 && sourceNode.textContent.length < 500;
|
|
|
|
const childNodes = [];
|
|
for (const child of sourceNode.childNodes) {
|
|
const childClone = cloneNode(child, keep || isSmallDropdown);
|
|
if (childClone) childNodes.push(childClone);
|
|
}
|
|
if (sourceNode.tagName === 'IFRAME') {
|
|
try {
|
|
const iDoc = sourceNode.contentDocument || sourceNode.contentWindow?.document;
|
|
if (iDoc && iDoc.body && iDoc.body.children.length > 0) {
|
|
const wrapper = document.createElement('div');
|
|
wrapper.setAttribute('data-iframe-content', sourceNode.src || '');
|
|
for (const ch of iDoc.body.childNodes) {
|
|
const c = cloneNode(ch, keep);
|
|
if (c) wrapper.appendChild(c);
|
|
}
|
|
if (wrapper.childNodes.length) childNodes.push(wrapper);
|
|
}
|
|
} catch(e) {}
|
|
}
|
|
if (sourceNode.shadowRoot) {
|
|
for (const shadowChild of sourceNode.shadowRoot.childNodes) {
|
|
const shadowClone = cloneNode(shadowChild, keep);
|
|
if (shadowClone) childNodes.push(shadowClone);
|
|
}
|
|
}
|
|
|
|
const rect = sourceNode.getBoundingClientRect();
|
|
const style = window.getComputedStyle(sourceNode);
|
|
const area = (style.display === 'none' || style.visibility === 'hidden' || parseFloat(style.opacity) <= 0)?0:rect.width * rect.height;
|
|
const isVisible = (rect.width > 1 && rect.height > 1 &&
|
|
style.display !== 'none' && style.visibility !== 'hidden' &&
|
|
parseFloat(style.opacity) > 0 &&
|
|
Math.abs(rect.left) < 5000 && Math.abs(rect.top) < 5000)
|
|
|| isSmallDropdown;
|
|
const zIndex = style.position !== 'static' ? (parseInt(style.zIndex) || 0) : 0;
|
|
|
|
let info = {
|
|
rect, area, isVisible, isSmallDropdown, zIndex,
|
|
style: {
|
|
display: style.display, visibility: style.visibility,
|
|
opacity: style.opacity, position: style.position
|
|
}};
|
|
|
|
const nonTextChildren = childNodes.filter(child => child.nodeType !== 3);
|
|
const hasValidChildren = nonTextChildren.length > 0;
|
|
|
|
if (hasValidChildren) {
|
|
const childrenInfos = nonTextChildren.map(c => nodeInfo.get(c)).filter(i => i && i.rect && i.rect.width > 0 && i.rect.height > 0);
|
|
const bgAlpha = (() => {
|
|
const c = style.backgroundColor;
|
|
if (!c || c === 'transparent') return 0;
|
|
const m = c.match(/rgba?\([^)]+,\s*([\d.]+)\)/);
|
|
return m ? parseFloat(m[1]) : 1;
|
|
})();
|
|
const hasVisualBg = bgAlpha > 0.1 || style.backgroundImage !== 'none' || (style.backdropFilter && style.backdropFilter !== 'none') || style.boxShadow !== 'none';
|
|
|
|
if (!hasVisualBg && childrenInfos.length > 0) {
|
|
// Skip fixed/absolute children when computing parent's merged rect (they're out of flow)
|
|
const flowChildren = childrenInfos.filter(cInfo => cInfo.style && cInfo.style.position !== 'fixed' && cInfo.style.position !== 'absolute');
|
|
if (flowChildren.length > 0) {
|
|
let minL = Infinity, minT = Infinity, maxR = -Infinity, maxB = -Infinity;
|
|
for (const cInfo of flowChildren) {
|
|
minL = Math.min(minL, cInfo.rect.left);
|
|
minT = Math.min(minT, cInfo.rect.top);
|
|
maxR = Math.max(maxR, cInfo.rect.right);
|
|
maxB = Math.max(maxB, cInfo.rect.bottom);
|
|
}
|
|
info.rect = { left: minL, top: minT, right: maxR, bottom: maxB, width: maxR - minL, height: maxB - minT };
|
|
info.area = info.rect.width * info.rect.height;
|
|
} else {
|
|
const maxC = childrenInfos.filter(i => i.isVisible).sort((a, b) => b.area - a.area)[0];
|
|
if (maxC && maxC.area > 10000 && (!isVisible || maxC.area > info.area * 5)) info = maxC;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (sourceNode.nodeType === 1 && sourceNode.tagName === 'DIV') {
|
|
if (!hasValidChildren && !sourceNode.textContent.trim()) return null;
|
|
}
|
|
// aria-hidden + not visible = truly hidden (e.g. mobile menus), remove even if has children
|
|
if (sourceNode.getAttribute && sourceNode.getAttribute('aria-hidden') === 'true' && !info.isVisible) {
|
|
return null;
|
|
}
|
|
if (info.isVisible || hasValidChildren || keep) {
|
|
childNodes.forEach(child => clone.appendChild(child));
|
|
return clone;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
domCopy: cloneNode(document.body),
|
|
getNodeInfo: node => nodeInfo.get(node),
|
|
isVisible: node => {
|
|
const info = nodeInfo.get(node);
|
|
return info && info.isVisible;
|
|
}
|
|
};
|
|
}
|
|
const { domCopy, getNodeInfo, isVisible } = createEnhancedDOMCopy();
|
|
if (text_only) {
|
|
const blocks = new Set(['DIV','P','H1','H2','H3','H4','H5','H6','LI','TR','SECTION','ARTICLE','HEADER','FOOTER','NAV','BLOCKQUOTE','PRE','HR','BR','DT','DD','FIGCAPTION','DETAILS','SUMMARY']);
|
|
domCopy.querySelectorAll('*').forEach(el => {
|
|
if (blocks.has(el.tagName)) el.insertAdjacentText('beforebegin', '\n');
|
|
});
|
|
domCopy.querySelectorAll('input:not([type=hidden]),textarea,select').forEach(el=>{
|
|
const p=[el.tagName,el.id&&'#'+el.id,el.getAttribute('name')&&'name='+el.getAttribute('name'),el.tagName==='INPUT'&&'type='+(el.getAttribute('type')||'text'),el.getAttribute('placeholder')&&'"'+el.getAttribute('placeholder')+'"',el.getAttribute('data-autofilled')&&'autofilled',el.disabled&&'disabled',el.tagName==='SELECT'&&el.getAttribute('data-selected')&&'="'+el.getAttribute('data-selected')+'"'].filter(Boolean).join(' ');
|
|
el.insertAdjacentText('beforebegin','\n['+p+']\n');
|
|
});
|
|
domCopy.querySelectorAll('button[disabled]').forEach(el=>el.insertAdjacentText('beforebegin','[DISABLED] '));
|
|
return domCopy.textContent;
|
|
}
|
|
const viewportArea = window.innerWidth * window.innerHeight;
|
|
|
|
function analyzeNode(node, pPathType='main') {
|
|
// 处理非元素节点和叶节点
|
|
if (node.nodeType !== 1 || !node.children.length) {
|
|
node.nodeType === 1 && (node.dataset.mark = 'K:leaf');
|
|
return;
|
|
}
|
|
const pathType = (node.dataset.mark === 'K:secondary') ? 'second' : pPathType;
|
|
const nodeInfoData = getNodeInfo(node);
|
|
if (!nodeInfoData || !nodeInfoData.rect) return;
|
|
const rectn = nodeInfoData.rect;
|
|
if (rectn.width < window.innerWidth * 0.8 && rectn.height < window.innerHeight * 0.8) return node;
|
|
if (node.tagName === 'TABLE') return;
|
|
const children = Array.from(node.children);
|
|
if (children.length === 1) {
|
|
node.dataset.mark = 'K:container';
|
|
return analyzeNode(children[0], pathType);
|
|
}
|
|
if (children.length > 10) return;
|
|
|
|
// 获取子元素信息并排序
|
|
const childrenInfo = children.map(child => {
|
|
const info = getNodeInfo(child) || { rect: {}, style: {} };
|
|
return { node: child, rect: info.rect, style: info.style,
|
|
area: info.area, zIndex: (info.zIndex || 0), isVisible: info.isVisible };
|
|
});
|
|
childrenInfo.sort((a, b) => b.area - a.area);
|
|
|
|
// 检测是划分还是覆盖
|
|
const isOverlay = hasOverlap(childrenInfo);
|
|
node.dataset.mark = isOverlay ? 'K:overlayParent' : 'K:partitionParent';
|
|
|
|
if (isOverlay) handleOverlayContainer(childrenInfo, pathType);
|
|
else handlePartitionContainer(childrenInfo, pathType);
|
|
|
|
console.log(`${isOverlay ? '覆盖' : '划分'}容器:`, node, `子元素数量: ${children.length}`);
|
|
console.log('子元素及标记:', children.map(child => ({
|
|
element: child,
|
|
mark: child.dataset.mark || '无',
|
|
info: getNodeInfo ? getNodeInfo(child) : undefined
|
|
})));
|
|
for (const child of children)
|
|
if (!child.dataset.mark || child.dataset.mark[0] !== 'R') analyzeNode(child, pathType);
|
|
}
|
|
|
|
// 处理划分容器
|
|
function handlePartitionContainer(childrenInfo, pathType) {
|
|
childrenInfo.sort((a, b) => b.area - a.area);
|
|
const totalArea = childrenInfo.reduce((sum, item) => sum + item.area, 0);
|
|
console.log(childrenInfo[0].area / totalArea);
|
|
const hasMainElement = childrenInfo.length >= 1 &&
|
|
(childrenInfo[0].area / totalArea > 0.5) &&
|
|
(childrenInfo.length === 1 || childrenInfo[0].area > childrenInfo[1].area * 2);
|
|
if (hasMainElement) {
|
|
childrenInfo[0].node.dataset.mark = 'K:main';
|
|
for (let i = 1; i < childrenInfo.length; i++) {
|
|
const child = childrenInfo[i];
|
|
let className = (child.node.getAttribute('class') || '').toLowerCase();
|
|
let isSecondary = containsButton(child.node);
|
|
if (className.includes('nav')) isSecondary = true;
|
|
if (className.includes('breadcrumbs')) isSecondary = true;
|
|
if (className.includes('header') && className.includes('table')) isSecondary = true;
|
|
if (child.node.innerHTML.trim().replace(/\s+/g, '').length < 500) isSecondary = true;
|
|
if (child.node.textContent.trim().length > 200) isSecondary = true; // P3: 有实质文本内容则保留
|
|
if (child.style.visibility === 'hidden') isSecondary = false;
|
|
if (isSecondary) child.node.dataset.mark = 'K:secondary';
|
|
else child.node.dataset.mark = 'K:nonEssential';
|
|
}
|
|
} else {
|
|
return; // relaxed: skip equalmany filtering, list truncation handles token budget
|
|
const uniqueClassNames = new Set(childrenInfo.map(item => item.node.getAttribute('class') || '')).size;
|
|
const highClassNameVariety = uniqueClassNames >= childrenInfo.length * 0.8;
|
|
if (pathType !== 'main' && highClassNameVariety && childrenInfo.length > 5) {
|
|
childrenInfo.forEach(child => child.node.dataset.mark = 'R:equalmany');
|
|
} else {
|
|
childrenInfo.forEach(child => child.node.dataset.mark = 'K:equal');
|
|
}
|
|
}
|
|
}
|
|
|
|
function containsButton(container) {
|
|
const hasStandardButton = container.querySelector('button, input[type="button"], input[type="submit"], [role="button"]') !== null;
|
|
if (hasStandardButton) return true;
|
|
const hasClassButton = container.querySelector('[class*="-btn"], [class*="-button"], .button, .btn, [class*="btn-"]') !== null;
|
|
return hasClassButton;
|
|
}
|
|
|
|
function handleOverlayContainer(childrenInfo, pathType) {
|
|
// elementFromPoint ground truth: 让浏览器告诉我们谁在视觉最上层
|
|
const _efp = document.elementFromPoint(window.innerWidth/2, window.innerHeight/2);
|
|
if (_efp) { let _el = _efp; while (_el) { const _h = childrenInfo.find(c => c.node.id && c.node.id === _el.id); if (_h) { _h.zIndex = 9999; break; } _el = _el.parentElement; } }
|
|
const sorted = [...childrenInfo].sort((a, b) => b.zIndex - a.zIndex);
|
|
console.log('排序后的子元素:', sorted);
|
|
if (sorted.length === 0) return;
|
|
|
|
const top = sorted[0];
|
|
const rect = top.rect;
|
|
const topNode = top.node;
|
|
const isComplex = top.node.querySelectorAll('input, select, textarea, button, a, [role="button"]').length >= 1;
|
|
|
|
const textContent = topNode.textContent?.trim() || '';
|
|
const textLength = textContent.length;
|
|
const hasLinks = topNode.querySelectorAll('a').length > 0;
|
|
const isMostlyText = textLength > 7 && !hasLinks;
|
|
|
|
const centerDiff = Math.abs((rect.left + rect.width/2) - window.innerWidth/2) / window.innerWidth;
|
|
const minDimensionRatio = Math.min(rect.width / window.innerWidth, rect.height / window.innerHeight);
|
|
const maxDimensionRatio = Math.max(rect.width / window.innerWidth, rect.height / window.innerHeight);
|
|
const isNearTop = rect.top < 50;
|
|
const isDialog = (top.node.querySelector('iframe') || top.node.querySelector('button') || top.node.querySelector('input')) && centerDiff < 0.3;
|
|
|
|
if (isComplex && centerDiff < 0.2 &&
|
|
((minDimensionRatio > 0.2 && rect.width/window.innerWidth < 0.98) || minDimensionRatio > 0.95)) {
|
|
top.node.dataset.mark = 'K:mainInteractive';
|
|
sorted.slice(1).forEach(e => {
|
|
if ((parseInt(e.zIndex)||0) <= (parseInt(sorted[0].zIndex)||0)) {
|
|
e.node.dataset.mark = 'R:covered';
|
|
} else {
|
|
e.node.dataset.mark = 'K:noncovered';
|
|
}
|
|
});
|
|
} else {
|
|
if (isComplex && isNearTop && maxDimensionRatio > 0.4 && top.isVisible) {
|
|
top.node.dataset.mark = 'K:topBar';
|
|
} else if (isMostlyText || isComplex || isDialog) {
|
|
topNode.dataset.mark = 'K:messageContent';
|
|
} else {
|
|
topNode.dataset.mark = 'R:floatingAd';
|
|
}
|
|
const rest = sorted.slice(1);
|
|
rest.length && (!hasOverlap(rest) ? handlePartitionContainer(rest, pathType) : handleOverlayContainer(rest, pathType));
|
|
}
|
|
}
|
|
|
|
function hasOverlap(items) {
|
|
return items.some((a, i) =>
|
|
items.slice(i+1).some(b => {
|
|
const r1 = a.rect, r2 = b.rect;
|
|
if (!r1.width || !r2.width || !r1.height || !r2.height) {return false;}
|
|
const epsilon = 1;
|
|
const x1 = r1.x !== undefined ? r1.x : r1.left;
|
|
const y1 = r1.y !== undefined ? r1.y : r1.top;
|
|
const x2 = r2.x !== undefined ? r2.x : r2.left;
|
|
const y2 = r2.y !== undefined ? r2.y : r2.top;
|
|
return !(x1 + r1.width <= x2 + epsilon || x1 >= x2 + r2.width - epsilon ||
|
|
y1 + r1.height <= y2 + epsilon || y1 >= y2 + r2.height - epsilon
|
|
);
|
|
})
|
|
);
|
|
}
|
|
|
|
// Hoist top 1-2 deep fixed dialogs to body level for overlay detection
|
|
const _fc = [...domCopy.querySelectorAll('*')].filter(el => {
|
|
if (el.parentNode === domCopy) return false;
|
|
const info = getNodeInfo(el);
|
|
if (!info?.rect || info.style.position !== 'fixed') return false;
|
|
const r = info.rect, cover = (r.width * r.height) / viewportArea;
|
|
const cd = Math.abs((r.left + r.width/2) - window.innerWidth/2) / window.innerWidth;
|
|
return cover > 0.15 && cover < 1.0 && cd < 0.3 && el.querySelector('button, input, a, [role="button"], iframe');
|
|
}).filter((el, _, arr) => !arr.some(o => o !== el && o.contains(el)))
|
|
.sort((a, b) => (getNodeInfo(b).rect.width * getNodeInfo(b).rect.height) - (getNodeInfo(a).rect.width * getNodeInfo(a).rect.height))
|
|
.slice(0, 2);
|
|
_fc.forEach(el => { const r = getNodeInfo(el).rect; console.log('[simphtml] Hoisted fixed dialog:', el.tagName + (el.id ? '#'+el.id : '') + (el.className ? '.'+String(el.className).split(' ')[0] : ''), Math.round(r.width)+'x'+Math.round(r.height), Math.round(100*r.width*r.height/viewportArea)+'%'); el.parentNode.removeChild(el); domCopy.appendChild(el); });
|
|
const result = analyzeNode(domCopy);
|
|
domCopy.querySelectorAll('[data-mark^="R:"]').forEach(el=>el.parentNode?.removeChild(el));
|
|
let root = domCopy;
|
|
while (root.children.length === 1) {
|
|
root = root.children[0];
|
|
}
|
|
for (let ii = 0; ii < 3; ii++) {
|
|
root.querySelectorAll('div').forEach(div => (!div.textContent.trim() && div.children.length === 0) && div.remove());
|
|
}
|
|
root.querySelectorAll('[data-mark]').forEach(e => e.removeAttribute('data-mark'));
|
|
root.removeAttribute('data-mark');
|
|
root.querySelectorAll('iframe').forEach(f => {
|
|
if (f.children.length) {
|
|
const d = document.createElement('div');
|
|
for (const a of f.attributes) d.setAttribute(a.name, a.value);
|
|
d.setAttribute('data-tag', 'iframe');
|
|
while (f.firstChild) d.appendChild(f.firstChild);
|
|
f.parentNode.replaceChild(d, f);
|
|
}
|
|
});
|
|
return root.outerHTML;
|
|
}
|
|
optHTML()'''
|
|
|
|
js_findMainList = r'''function findMainList(startElement = null) {
|
|
const root = startElement || document.body;
|
|
const MIN_CHILDREN = 8;
|
|
const MAX_CONTAINERS = 20;
|
|
|
|
// 全局扫描：收集候选容器，按 l1 + l2*0.1 排序（l2=孙子元素数，捕获表格等多层结构）
|
|
const candidates = [];
|
|
const allEls = root.querySelectorAll('*');
|
|
for (const node of allEls) {
|
|
if (node.closest('svg')) continue;
|
|
const l1 = node.children.length;
|
|
if (l1 < 5) continue;
|
|
let l2 = 0;
|
|
for (const child of node.children) l2 += child.children.length;
|
|
const score = l1 + l2 * 0.1;
|
|
if (score >= MIN_CHILDREN) candidates.push({node, score});
|
|
}
|
|
candidates.sort((a, b) => b.score - a.score);
|
|
const toProcess = candidates.slice(0, MAX_CONTAINERS).map(c => c.node);
|
|
|
|
// 对每个容器找候选组并评分
|
|
let allCandidates = [];
|
|
for (const container of toProcess) {
|
|
const topGroups = findTopGroups(container, 3);
|
|
for (const groupInfo of topGroups) {
|
|
const items = findMatchingElements(container, groupInfo.selector);
|
|
if (items.length >= 5) {
|
|
const score = scoreContainer(container, items) + groupInfo.score;
|
|
if (score >= 30) {
|
|
allCandidates.push({ container, selector: groupInfo.selector, items, score });
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// 按分数降序排列
|
|
allCandidates.sort((a, b) => b.score - a.score);
|
|
|
|
// 去重：移除与更高分候选重叠超50%的结果
|
|
const kept = [];
|
|
for (const cand of allCandidates) {
|
|
let dominated = false;
|
|
for (const k of kept) {
|
|
if (k.container.contains(cand.container) || cand.container.contains(k.container)) {
|
|
const kSet = new Set(k.items);
|
|
const overlap = cand.items.filter(it => kSet.has(it)).length;
|
|
if (overlap > cand.items.length * 0.5) { dominated = true; break; }
|
|
}
|
|
}
|
|
if (!dominated) kept.push(cand);
|
|
}
|
|
|
|
function describeResult(container, items, selector, score) {
|
|
if(container&&!container.id)container.id='_ljq'+(window._lci=(window._lci||0)+1);
|
|
const cTag = container ? container.tagName : null;
|
|
const cId = container ? (container.id || '') : '';
|
|
const cClass = container ? (String(container.className || '').trim()) : '';
|
|
const result = {
|
|
containerTag: cTag, containerId: cId, containerClass: cClass,
|
|
itemCount: items.length,
|
|
};
|
|
let prefix = '';
|
|
if (cId) prefix = '#' + CSS.escape(cId);
|
|
if (selector) result.selector = prefix ? (prefix + ' > ' + selector) : selector;
|
|
if (score !== undefined) result.score = score;
|
|
if (items.length > 0) {
|
|
result.firstItemPreview = items[0].outerHTML.substring(0, 200);
|
|
result.itemTags = items.slice(0, 10).map(el => el.tagName + (el.className ? '.' + String(el.className).trim().split(/\s+/)[0] : ''));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
if (kept.length === 0) return [];
|
|
|
|
return kept.map(c => describeResult(c.container, c.items, c.selector, c.score));
|
|
}
|
|
|
|
function findTopGroups(container, limit) {
|
|
const children = Array.from(container.children).filter(c => !c.closest('svg'));
|
|
const totalChildren = children.length;
|
|
if (totalChildren < 3) return [];
|
|
|
|
const minGroupSize = Math.max(3, Math.floor(totalChildren * 0.2));
|
|
const groups = [];
|
|
|
|
// 统计标签和类名
|
|
const tagFreq = {}, classFreq = {}, tagMap = {}, classMap = {};
|
|
|
|
children.forEach(child => {
|
|
// 统计标签
|
|
const tag = child.tagName.toLowerCase();
|
|
if (tag === "td") return;
|
|
tagFreq[tag] = (tagFreq[tag] || 0) + 1;
|
|
if (!tagMap[tag]) tagMap[tag] = [];
|
|
tagMap[tag].push(child);
|
|
|
|
// 统计类名
|
|
if (child.className) {
|
|
child.className.trim().split(/\s+/).forEach(cls => {
|
|
if (cls) {
|
|
classFreq[cls] = (classFreq[cls] || 0) + 1;
|
|
if (!classMap[cls]) classMap[cls] = [];
|
|
classMap[cls].push(child);
|
|
}
|
|
});
|
|
}
|
|
});
|
|
|
|
// 评分函数
|
|
const scoreGroup = (selector, elements) => {
|
|
const coverage = elements.length / totalChildren;
|
|
let specificity = selector.startsWith('.')
|
|
? (0.6 + (selector.match(/\./g).length - 1) * 0.1) // 类选择器
|
|
: (selector.includes('.')
|
|
? (0.7 + (selector.match(/\./g).length) * 0.1) // 标签+类
|
|
: 0.3); // 纯标签
|
|
return (coverage * 0.5) + (specificity * 0.5);
|
|
};
|
|
|
|
// 添加标签组
|
|
Object.keys(tagFreq).forEach(tag => {
|
|
if (tag !== "div" && tagFreq[tag] >= minGroupSize) {
|
|
groups.push({
|
|
selector: tag,
|
|
elements: tagMap[tag],
|
|
score: scoreGroup(tag, tagMap[tag]) - 0.5
|
|
});
|
|
}
|
|
});
|
|
|
|
// 添加类组
|
|
Object.keys(classFreq).forEach(cls => {
|
|
if (classFreq[cls] >= minGroupSize) {
|
|
const selector = '.' + CSS.escape(cls);
|
|
groups.push({
|
|
selector,
|
|
elements: classMap[cls],
|
|
score: scoreGroup(selector, classMap[cls])
|
|
});
|
|
}
|
|
});
|
|
// 添加标签+类组合
|
|
const topTags = Object.keys(tagFreq).filter(t => tagFreq[t] >= minGroupSize).slice(0, 3);
|
|
const topClasses = Object.keys(classFreq).filter(c => classFreq[c] >= minGroupSize).sort((a, b) => classFreq[b] - classFreq[a]).slice(0, 3);
|
|
|
|
// 标签+类
|
|
topTags.forEach(tag => {
|
|
topClasses.forEach(cls => {
|
|
const elements = children.filter(el =>
|
|
el.tagName.toLowerCase() === tag &&
|
|
el.className && el.className.split(/\s+/).includes(cls)
|
|
);
|
|
|
|
if (elements.length >= minGroupSize) {
|
|
const selector = tag + '.' + CSS.escape(cls);
|
|
groups.push({selector, elements, score: scoreGroup(selector, elements)});
|
|
}
|
|
});
|
|
});
|
|
|
|
// 多类组合
|
|
for (let i = 0; i < topClasses.length; i++) {
|
|
for (let j = i + 1; j < topClasses.length; j++) {
|
|
const elements = children.filter(el =>
|
|
el.className && el.className.split(/\s+/).includes(topClasses[i]) && el.className.split(/\s+/).includes(topClasses[j]));
|
|
|
|
if (elements.length >= minGroupSize) {
|
|
const selector = '.' + CSS.escape(topClasses[i]) + '.' + CSS.escape(topClasses[j]);
|
|
groups.push({selector, elements,score: scoreGroup(selector, elements)});
|
|
}
|
|
}
|
|
}
|
|
// 返回得分最高的N个组
|
|
return groups.sort((a, b) => b.score - a.score).slice(0, limit);
|
|
}
|
|
|
|
function findMatchingElements(container, selector) {
|
|
try {
|
|
return Array.from(container.querySelectorAll(selector));
|
|
} catch (e) {
|
|
// 处理无效选择器
|
|
console.error('Invalid selector:', selector, e);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function scoreContainer(container, items) {
|
|
if (!container || items.length < 3) return 0;
|
|
// 1. 计算基础面积数据
|
|
const containerRect = container.getBoundingClientRect();
|
|
const containerArea = containerRect.width * containerRect.height;
|
|
if (containerArea < 10000) return 0; // 容器太小
|
|
|
|
// 收集列表项面积数据
|
|
const itemAreas = [];
|
|
let totalItemArea = 0;
|
|
let visibleItems = 0;
|
|
|
|
items.forEach(item => {
|
|
const rect = item.getBoundingClientRect();
|
|
const area = rect.width * rect.height;
|
|
if (area > 0) {
|
|
totalItemArea += area;
|
|
itemAreas.push(area);
|
|
visibleItems++;
|
|
}
|
|
});
|
|
// 如果可见项太少，返回低分
|
|
if (visibleItems < 3) return 0;
|
|
// 防止异常值：确保面积不超过容器
|
|
totalItemArea = Math.min(totalItemArea, containerArea * 0.98);
|
|
const areaRatio = totalItemArea / containerArea;
|
|
// 3. 计算各项评分 - 使用线性插值而非阶梯
|
|
// 3.2 面积比评分 - 最多40分，连续曲线
|
|
// 使用sigmoid函数让评分更平滑
|
|
const areaScore = 40 / (1 + Math.exp(-12 * (areaRatio - 0.4)));
|
|
|
|
// 3.3 均匀性评分 - 最多20分，连续曲线
|
|
let uniformityScore = 0;
|
|
if (itemAreas.length >= 3) {
|
|
const mean = itemAreas.reduce((sum, area) => sum + area, 0) / itemAreas.length;
|
|
const variance = itemAreas.reduce((sum, area) => sum + Math.pow(area - mean, 2), 0) / itemAreas.length;
|
|
const cv = mean > 0 ? Math.sqrt(variance) / mean : 1;
|
|
// 指数衰减函数，cv越小分数越高
|
|
uniformityScore = 20 * Math.exp(-2.5 * cv);
|
|
}
|
|
|
|
const baseScore = Math.log2(visibleItems) * 5 + Math.floor(visibleItems / 5) * 0.25;
|
|
const rawCountScore = Math.min(40, baseScore);
|
|
const countScore = rawCountScore * Math.max(0.1, uniformityScore / 20);
|
|
|
|
// 3.4 容器尺寸评分 - 最多15分，连续曲线
|
|
const viewportArea = window.innerWidth * window.innerHeight;
|
|
const containerViewportRatio = containerArea / viewportArea;
|
|
const sizeScore = 2 * (1 - 1/(1 + Math.exp(-10 * (containerViewportRatio - 0.25))));
|
|
|
|
let layoutScore = 0;
|
|
if (items.length >= 3) {
|
|
// 坐标分组并计算行列数
|
|
const uniqueRows = new Set(items.map(item => Math.round(item.getBoundingClientRect().top / 5) * 5)).size;
|
|
const uniqueCols = new Set(items.map(item => Math.round(item.getBoundingClientRect().left / 5) * 5)).size;
|
|
// 如果是单行或单列，直接给满分；否则评估网格质量
|
|
if (uniqueRows === 1 || uniqueCols === 1) { layoutScore = 20;
|
|
} else {
|
|
const coverage = Math.min(1, items.length / (uniqueRows * uniqueCols));
|
|
const efficiency = Math.max(0, 1 - (uniqueRows + uniqueCols) / (2 * items.length));
|
|
layoutScore = 20 * (0.7 * coverage + 0.3 * efficiency);
|
|
}
|
|
}
|
|
|
|
// 总分 - 仍然保持100分左右的总分
|
|
const totalScore = countScore + areaScore + uniformityScore + layoutScore + sizeScore;
|
|
|
|
if (totalScore > 100)
|
|
console.log(container, {
|
|
total: totalScore.toFixed(2),
|
|
count: countScore.toFixed(2),
|
|
areaRatio: areaRatio.toFixed(2),
|
|
area: areaScore.toFixed(2),
|
|
uniformity: uniformityScore.toFixed(2),
|
|
size: sizeScore.toFixed(2),
|
|
layout: layoutScore.toFixed(2)
|
|
});
|
|
|
|
return totalScore;
|
|
}'''
|
|
|
|
def optimize_html_for_tokens(html):
|
|
if type(html) is str: soup = BeautifulSoup(html, 'html.parser')
|
|
else: soup = html
|
|
for svg in soup.find_all('svg'):
|
|
svg.clear(); svg.attrs = {}
|
|
[tag.attrs.pop('style', None) for tag in soup.find_all(True)]
|
|
for tag in soup.find_all(True):
|
|
if tag.has_attr('src'):
|
|
if tag['src'].startswith('data:'): tag['src'] = '__img__'
|
|
elif len(tag['src']) > 30: tag['src'] = '__url__'
|
|
if tag.has_attr('href') and len(tag['href']) > 30: tag['href'] = '__link__'
|
|
if tag.has_attr('action') and len(tag['action']) > 30: tag['action'] = '__url__'
|
|
for a in ('value', 'title', 'alt'):
|
|
if tag.has_attr(a) and isinstance(tag[a], str) and len(tag[a]) > 100: tag[a] = tag[a][:50] + ' ...'
|
|
for attr in list(tag.attrs.keys()):
|
|
if attr not in ['id', 'class', 'name', 'src', 'href', 'alt', 'value', 'type', 'placeholder',
|
|
'disabled', 'checked', 'selected', 'readonly', 'required', 'multiple',
|
|
'role', 'aria-label', 'aria-expanded', 'aria-hidden', 'contenteditable',
|
|
'title', 'for', 'action', 'method', 'target', 'colspan', 'rowspan']:
|
|
if attr.startswith('data-v'): tag.attrs.pop(attr, None)
|
|
elif attr.startswith('data-') and isinstance(tag[attr], str) and len(tag[attr]) > 20:
|
|
tag[attr] = '__data__'
|
|
elif not attr.startswith('data-'): tag.attrs.pop(attr, None)
|
|
return soup
|
|
|
|
|
|
temp_monitor_js = """function startStrMonitor(interval) {
|
|
if (window._tm && window._tm.id) clearInterval(window._tm.id);
|
|
window._tm = {extract: () => {
|
|
const texts = new Set(), walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
|
|
let node, t, s; while (node = walker.nextNode())
|
|
((t = node.textContent.trim()) && t.length > 10 && !(s = t.substring(0, 20)).includes('_')) && texts.add(s);
|
|
return texts;
|
|
}};
|
|
window._tm.init = window._tm.extract();
|
|
window._tm.all = new Set();
|
|
window._tm.id = setInterval(() => window._tm.extract().forEach(t => window._tm.all.add(t)), interval);
|
|
}
|
|
startStrMonitor(450);
|
|
"""
|
|
def start_temp_monitor(driver):
|
|
try: driver.execute_js(temp_monitor_js)
|
|
except: pass
|
|
|
|
def get_temp_texts(driver):
|
|
js = """function stopStrMonitor() {
|
|
if (!window._tm) return [];
|
|
clearInterval(window._tm.id);
|
|
const final = window._tm.extract();
|
|
const newlySeen = [...window._tm.all].filter(t => !window._tm.init.has(t));
|
|
let result;
|
|
if (newlySeen.length < 8) {
|
|
result = newlySeen;
|
|
} else {
|
|
result = newlySeen.filter(t => !final.has(t));
|
|
}
|
|
delete window._tm;
|
|
return result;
|
|
}
|
|
stopStrMonitor();
|
|
"""
|
|
try: return list(set(driver.execute_js(js).get('data', [])))
|
|
except Exception as e:
|
|
print(e)
|
|
return []
|
|
|
|
import time, re, os
|
|
def get_main_block(driver, extra_js="", text_only=False):
|
|
page = driver.execute_js(f"{extra_js}\n{js_optHTML}\nreturn optHTML({str(text_only).lower()});").get('data', '')
|
|
if text_only:
|
|
page = re.sub(r' {2,}', ' ', page) # 连续空格→单空格
|
|
page = re.sub(r'^ +', '', page, flags=re.M) # 去行首空格
|
|
page = re.sub(r'(\n\s*){3,}', '\n\n', page) # 3+空行→1空行
|
|
return page.strip()
|
|
return page
|
|
|
|
def find_changed_elements(before_html, after_html):
|
|
before_soup = BeautifulSoup(before_html, 'html.parser')
|
|
after_soup = BeautifulSoup(after_html, 'html.parser')
|
|
def direct_text(el):
|
|
return ''.join(t.strip() for t in el.find_all(string=True, recursive=False)).strip()
|
|
def get_sig(el):
|
|
attrs = {k:v for k,v in el.attrs.items() if k != 'data-track-id'}
|
|
return f"{el.name}:{attrs}:{direct_text(el)}"
|
|
def build_sigs(soup):
|
|
result = {}
|
|
for el in soup.find_all(True):
|
|
sig = get_sig(el)
|
|
result.setdefault(sig, []).append(el)
|
|
return result
|
|
before_sigs, after_sigs = build_sigs(before_soup), build_sigs(after_soup)
|
|
changed = []
|
|
for sig, els in after_sigs.items():
|
|
if sig not in before_sigs: changed.extend(els)
|
|
elif len(els) > len(before_sigs[sig]): changed.extend(els[:len(els) - len(before_sigs[sig])])
|
|
if len(changed) == 0 and str(before_soup) != str(after_soup):
|
|
before_els, after_els = before_soup.find_all(True), after_soup.find_all(True)
|
|
for i in range(min(len(before_els), len(after_els))):
|
|
if get_sig(before_els[i]) != get_sig(after_els[i]): changed.append(after_els[i])
|
|
# 变化边界: parent不在changed中的元素
|
|
cids = set(id(el) for el in changed)
|
|
boundaries = [el for el in changed if el.parent is None or id(el.parent) not in cids]
|
|
top = max(boundaries, key=lambda el: len(str(el))) if boundaries else None
|
|
result = {"changed": len(changed)}
|
|
if top:
|
|
h = str(top)
|
|
result["top_change"] = h if len(h) <= 2000 else h[:2000] + '...[TRUNCATED]'
|
|
return result
|
|
|
|
def get_html(driver, cutlist=False, maxchars=35000, instruction="", extra_js="", text_only=False):
|
|
if cutlist: rr = driver.execute_js(js_findMainList + "return findMainList(document.body);").get('data', [])
|
|
page = get_main_block(driver, extra_js=extra_js, text_only=text_only)
|
|
if text_only: return page
|
|
soup = optimize_html_for_tokens(page)
|
|
for div in soup.select('div[data-tag="iframe"]'):
|
|
div.name = 'iframe'; del div['data-tag']
|
|
html = str(soup)
|
|
if not cutlist: return html
|
|
lists = rr if isinstance(rr, list) else ([rr] if isinstance(rr, dict) and rr.get('selector') else [])
|
|
if lists: print(f"[cutlist] Found {len(lists)} list(s): {[e.get('selector','?') if isinstance(e,dict) else '?' for e in lists]}")
|
|
for entry in lists:
|
|
sel = entry.get('selector') if isinstance(entry, dict) else None
|
|
if not sel: continue
|
|
try: items = soup.select(sel)
|
|
except Exception: print(f'[cutlist] skip invalid selector: {sel}'); continue
|
|
if len(items) < 5: continue
|
|
total_len = sum(len(str(it)) for it in items)
|
|
avg_len = total_len / len(items)
|
|
print(f"[cutlist] '{sel}': {len(items)} items, avg {avg_len:.0f} chars, total {total_len}, if keep 3, save ~{total_len - 3 * avg_len:.0f} chars")
|
|
if avg_len < 200 or (avg_len < 700 and total_len < 2500): continue
|
|
hit = [it for it in items if instruction and instruction.strip() and instruction in it.get_text(" ",strip=True)]
|
|
keep = hit[:6] if hit else items[:3]
|
|
removed = [it for it in items if it not in keep]
|
|
sample_texts = []
|
|
for rm in removed[:5]:
|
|
txt = rm.get_text(" ", strip=True)[:40]
|
|
if txt: sample_texts.append(txt)
|
|
hint_parts = [f'[FAKE ELEMENT] {len(removed)} more items hidden, selector: "{sel}"']
|
|
if sample_texts: hint_parts.append('Hidden items: ' + ','.join(f'"{t}"' for t in sample_texts))
|
|
hint_tag = soup.new_tag("div")
|
|
hint_tag.string = ' '.join(hint_parts)
|
|
if keep: keep[-1].insert_after(hint_tag)
|
|
for it in removed: it.decompose()
|
|
ss = str(optimize_html_for_tokens(soup)) if lists else html
|
|
print(f"[get_html] Result: {len(html)} -> {len(ss)} chars after cutlist ({100-len(ss)*100//len(html)}% saved)")
|
|
if len(ss) > maxchars: ss = str(smart_truncate(soup, maxchars))
|
|
return ss
|
|
|
|
def smart_truncate(soup, budget, _depth=0):
|
|
"""原地截断 soup 使其接近 budget 字符。
|
|
策略：穿透单子元素找分叉点；top3 能扛住 over 则按比例分担，否则从尾部删子元素。"""
|
|
CUT_THRESHOLD = 8000 # 小于此值直接去尾，大于则继续递归找分叉点
|
|
indent = ' ' * _depth
|
|
def cut(ele, keep):
|
|
from bs4 import NavigableString
|
|
s = str(ele)
|
|
over = len(s) - keep
|
|
if over <= 0: return
|
|
# 保护 FAKE ELEMENT 提示标签
|
|
protected = [c.extract() for c in ele.find_all(lambda tag: tag.string and '[FAKE ELEMENT]' in tag.string)]
|
|
s = str(ele)
|
|
over = len(s) - keep
|
|
if over <= 0:
|
|
for p in protected: ele.append(p)
|
|
return
|
|
marker = f' [TRUNCATED {over//1000}k chars]'
|
|
inner = ele.decode_contents()
|
|
tag_overhead = len(s) - len(inner)
|
|
inner_keep = max(keep - tag_overhead - len(marker), 0)
|
|
ele.clear()
|
|
if inner_keep > 0:
|
|
ele.append(BeautifulSoup(inner[:inner_keep], 'html.parser'))
|
|
ele.append(NavigableString(marker))
|
|
for p in protected: ele.append(p)
|
|
total = len(str(soup))
|
|
if total <= budget: return soup
|
|
kids = [(c, len(str(c))) for c in soup.children if c.name and not (c.string and '[FAKE ELEMENT]' in c.string)]
|
|
if not kids: return soup
|
|
selflen = total - sum(l for _, l in kids)
|
|
remaining_budget = max(budget - selflen, 0)
|
|
tag = getattr(soup, 'name', '?')
|
|
print(f'{indent}[smart_truncate] <{tag}> total={total} budget={budget} selflen={selflen} kids={len(kids)}')
|
|
# === 1 kid: 穿透 ===
|
|
if len(kids) == 1:
|
|
print(f'{indent} -> single child, recurse into <{kids[0][0].name}>')
|
|
smart_truncate(kids[0][0], remaining_budget, _depth)
|
|
return soup
|
|
over = sum(l for _, l in kids) - remaining_budget
|
|
if over <= 0: return soup
|
|
# 看 top 3 能否承担 over
|
|
ranked = sorted(range(len(kids)), key=lambda i: kids[i][1], reverse=True)
|
|
tops = list(ranked[:min(3, len(ranked))])
|
|
top_total = sum(kids[i][1] for i in tops)
|
|
if top_total < over:
|
|
# === top 3 扛不住，从尾部删子元素 ===
|
|
removed = 0
|
|
removed_count = 0
|
|
while kids and removed < over:
|
|
c, l = kids.pop(); c.decompose()
|
|
removed += l; removed_count += 1
|
|
print(f'{indent} -> tail-cut: removed {removed_count} children ({removed//1000}k chars) from end')
|
|
return soup
|
|
# === top 2-3 按比例分担 ===
|
|
# 过滤掉太小的 kid（不到最大的 10%），让大的全扛
|
|
max_size = kids[ranked[0]][1]
|
|
filtered = [i for i in tops if kids[i][1] >= max_size * 0.1]
|
|
filtered_total = sum(kids[i][1] for i in filtered)
|
|
if filtered_total >= over:
|
|
tops, top_total = filtered, filtered_total
|
|
# 先打印所有分配计划
|
|
actions = []
|
|
for i in tops:
|
|
c, l = kids[i]
|
|
share = int(over * l / top_total)
|
|
new_keep = l - share
|
|
print(f'{indent} -> <{c.name}> {l} -> {new_keep} (share={share})')
|
|
actions.append((c, l, new_keep))
|
|
# 再统一执行
|
|
for c, l, new_keep in actions:
|
|
if new_keep <= 0: c.decompose()
|
|
elif new_keep > CUT_THRESHOLD: smart_truncate(c, new_keep, _depth + 1)
|
|
else: cut(c, new_keep)
|
|
return soup
|
|
|
|
def execute_js_rich(script, driver, no_monitor=False):
|
|
last_html = None
|
|
if not no_monitor:
|
|
try: last_html = get_html(driver, cutlist=False, extra_js=temp_monitor_js, maxchars=9999999)
|
|
except: pass
|
|
result = None; error_msg = None; reloaded = False; newTabs = []
|
|
before_sids = set(driver.get_session_dict().keys()); response = {}
|
|
try:
|
|
print(f"Executing: {script[:250]} ...")
|
|
response = driver.execute_js(script)
|
|
result = response['data'] if 'data' in response else response.get('result')
|
|
if response.get('closed', 0) == 1: reloaded = True
|
|
time.sleep(1)
|
|
except Exception as e:
|
|
error = e.args[0] if e.args else str(e)
|
|
if isinstance(error, dict): error.pop('stack', None)
|
|
error_msg = str(error)
|
|
print(f"Error: {error_msg}")
|
|
rr = {
|
|
"status": "failed" if error_msg else "success",
|
|
"js_return": result,
|
|
"tab_id": driver.default_session_id
|
|
}
|
|
if reloaded: rr['reloaded'] = reloaded
|
|
if response.get('newTabs'): rr['newTabs'] = response['newTabs']
|
|
else:
|
|
after = driver.get_session_dict()
|
|
new_sids = {k: v for k, v in after.items() if k not in before_sids}
|
|
if new_sids:
|
|
newTabs = [{'id': k, 'url': v} for k, v in new_sids.items()]
|
|
rr['newTabs'] = newTabs
|
|
rr['suggestion'] = "页面已刷新，以上新标签页在执行期间连接。"
|
|
if error_msg: rr['error'] = error_msg
|
|
if no_monitor: return rr
|
|
if not reloaded:
|
|
try: rr['transients'] = get_temp_texts(driver)
|
|
except: rr['transients'] = []
|
|
if not reloaded and len(newTabs) == 0:
|
|
try:
|
|
current_html = get_html(driver, cutlist=False, maxchars=9999999)
|
|
if last_html is None: raise Exception("no baseline")
|
|
diff_data = find_changed_elements(last_html, current_html)
|
|
change_count = diff_data.get('changed', 0)
|
|
top_change = diff_data.get('top_change', '')
|
|
diff_summary = f"DOM变化量: {change_count}"
|
|
if top_change: diff_summary += f"\n最显著变化:\n{top_change}"
|
|
transients = rr.get('transients', [])
|
|
if change_count == 0 and not transients and len(newTabs) == 0:
|
|
diff_summary += " (页面无变化)"
|
|
rr['suggestion'] = "页面无明显变化"
|
|
except:
|
|
diff_summary = "页面变化监控不可用"
|
|
rr['diff'] = diff_summary
|
|
return rr
|