chore: import upstream snapshot with attribution
Sync browser-act to plugin repo / sync (push) Failing after 0s
Sync browser-act to plugin repo / sync (push) Failing after 0s
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,197 @@
|
||||
name: Community activity
|
||||
|
||||
on:
|
||||
issues:
|
||||
types: [opened]
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_target:
|
||||
types: [opened]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
relay:
|
||||
name: Relay activity
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event_name != 'pull_request_review' || (github.event.review.body != null && github.event.review.body != '') }}
|
||||
steps:
|
||||
- name: Send activity update
|
||||
env:
|
||||
ACTIVITY_WEBHOOK_URL: ${{ secrets.COMMUNITY_ACTIVITY_WEBHOOK_URL }}
|
||||
ACTIVITY_WEBHOOK_SECRET: ${{ secrets.COMMUNITY_ACTIVITY_WEBHOOK_SECRET }}
|
||||
run: |
|
||||
node <<'NODE'
|
||||
const crypto = require('crypto');
|
||||
const fs = require('fs');
|
||||
|
||||
async function main() {
|
||||
const webhookUrl = process.env.ACTIVITY_WEBHOOK_URL;
|
||||
if (!webhookUrl) {
|
||||
console.log('Activity webhook is not configured. Skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
const eventName = process.env.GITHUB_EVENT_NAME;
|
||||
const eventPath = process.env.GITHUB_EVENT_PATH;
|
||||
if (!eventPath) {
|
||||
throw new Error('GITHUB_EVENT_PATH is not set.');
|
||||
}
|
||||
|
||||
const payload = JSON.parse(fs.readFileSync(eventPath, 'utf8'));
|
||||
const repo = payload.repository?.full_name || process.env.GITHUB_REPOSITORY || 'repository';
|
||||
|
||||
const shorten = (value, max = 800) => {
|
||||
const text = String(value || '').replace(/\s+/g, ' ').trim();
|
||||
return text.length > max ? `${text.slice(0, max - 1)}...` : text;
|
||||
};
|
||||
|
||||
const escapeMd = (value) => String(value || '')
|
||||
.replace(/\\/g, '\\\\')
|
||||
.replace(/\*/g, '\\*')
|
||||
.replace(/_/g, '\\_')
|
||||
.replace(/`/g, '\\`');
|
||||
|
||||
const userLogin = (user) => user?.login || 'unknown';
|
||||
|
||||
function activityFromPayload() {
|
||||
if (eventName === 'issues') {
|
||||
const issue = payload.issue;
|
||||
return {
|
||||
title: 'Issue created',
|
||||
actor: userLogin(issue.user),
|
||||
item: `#${issue.number} ${issue.title}`,
|
||||
url: issue.html_url,
|
||||
body: issue.body,
|
||||
};
|
||||
}
|
||||
|
||||
if (eventName === 'issue_comment') {
|
||||
const issue = payload.issue;
|
||||
const isPullRequest = Boolean(issue.pull_request);
|
||||
return {
|
||||
title: isPullRequest ? 'PR comment' : 'Issue comment',
|
||||
actor: userLogin(payload.comment?.user),
|
||||
item: `#${issue.number} ${issue.title}`,
|
||||
url: payload.comment?.html_url || issue.html_url,
|
||||
body: payload.comment?.body,
|
||||
};
|
||||
}
|
||||
|
||||
if (eventName === 'pull_request_target') {
|
||||
const pullRequest = payload.pull_request;
|
||||
return {
|
||||
title: 'PR created',
|
||||
actor: userLogin(pullRequest.user),
|
||||
item: `#${pullRequest.number} ${pullRequest.title}`,
|
||||
url: pullRequest.html_url,
|
||||
body: pullRequest.body,
|
||||
};
|
||||
}
|
||||
|
||||
if (eventName === 'pull_request_review') {
|
||||
const review = payload.review;
|
||||
const pullRequest = payload.pull_request;
|
||||
return {
|
||||
title: 'PR review',
|
||||
actor: userLogin(review.user),
|
||||
item: `#${pullRequest.number} ${pullRequest.title}`,
|
||||
url: review.html_url || pullRequest.html_url,
|
||||
body: review.body,
|
||||
};
|
||||
}
|
||||
|
||||
if (eventName === 'pull_request_review_comment') {
|
||||
const comment = payload.comment;
|
||||
const pullRequest = payload.pull_request;
|
||||
return {
|
||||
title: 'PR review comment',
|
||||
actor: userLogin(comment.user),
|
||||
item: `#${pullRequest.number} ${pullRequest.title}`,
|
||||
url: comment.html_url || pullRequest.html_url,
|
||||
body: comment.body,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
const activity = activityFromPayload();
|
||||
if (!activity) {
|
||||
console.log(`Unsupported event: ${eventName}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const lines = [
|
||||
`**Repository**: ${escapeMd(repo)}`,
|
||||
`**Actor**: ${escapeMd(activity.actor)}`,
|
||||
`**Target**: [${escapeMd(activity.item)}](${activity.url})`,
|
||||
];
|
||||
|
||||
const summary = shorten(activity.body);
|
||||
if (summary) {
|
||||
lines.push(`**Content**: ${escapeMd(summary)}`);
|
||||
}
|
||||
|
||||
const message = {
|
||||
msg_type: 'interactive',
|
||||
card: {
|
||||
header: {
|
||||
template: 'blue',
|
||||
title: {
|
||||
tag: 'plain_text',
|
||||
content: activity.title,
|
||||
},
|
||||
},
|
||||
elements: [
|
||||
{
|
||||
tag: 'markdown',
|
||||
content: lines.join('\n'),
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const secret = process.env.ACTIVITY_WEBHOOK_SECRET;
|
||||
if (secret) {
|
||||
const timestamp = Math.floor(Date.now() / 1000).toString();
|
||||
const sign = crypto
|
||||
.createHmac('sha256', `${timestamp}\n${secret}`)
|
||||
.update('')
|
||||
.digest('base64');
|
||||
message.timestamp = timestamp;
|
||||
message.sign = sign;
|
||||
}
|
||||
|
||||
const response = await fetch(webhookUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(message),
|
||||
});
|
||||
|
||||
const responseText = await response.text();
|
||||
|
||||
let result = {};
|
||||
try {
|
||||
result = JSON.parse(responseText);
|
||||
} catch (error) {
|
||||
// Non-JSON response body; fall back to HTTP status handling below.
|
||||
}
|
||||
|
||||
// Some webhook endpoints return HTTP 200 even on failure; a non-zero `code` means it was rejected.
|
||||
if (!response.ok || (result.code !== undefined && result.code !== 0)) {
|
||||
throw new Error(`Activity webhook failed (HTTP ${response.status}): ${responseText}`);
|
||||
}
|
||||
|
||||
console.log(`Activity webhook accepted: ${response.status}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
NODE
|
||||
@@ -0,0 +1,54 @@
|
||||
name: Sync browser-act to plugin repo
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'browser-act/**'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: sync-browser-act
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout source repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
path: source
|
||||
|
||||
- name: Checkout target repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: browser-act/claude-code-browser-act
|
||||
token: ${{ secrets.TARGET_REPO_TOKEN }}
|
||||
path: target
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Sync browser-act folder
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TARGET_DIR="target/plugins/browser-act/skills/browser-act"
|
||||
mkdir -p "$(dirname "$TARGET_DIR")"
|
||||
rm -rf "$TARGET_DIR"
|
||||
mkdir -p "$TARGET_DIR"
|
||||
rsync -a --delete source/browser-act/ "$TARGET_DIR/"
|
||||
|
||||
- name: Commit and push
|
||||
working-directory: target
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add -A plugins/browser-act/skills/browser-act
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes to sync."
|
||||
exit 0
|
||||
fi
|
||||
SHORT_SHA="${GITHUB_SHA::7}"
|
||||
git commit -m "sync: browser-act from ${GITHUB_REPOSITORY}@${SHORT_SHA}"
|
||||
git push origin HEAD:main
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
.DS_Store
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
.env
|
||||
.idea/
|
||||
.vscode/
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
output/
|
||||
*.log
|
||||
.omc
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 BrowserAct
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,182 @@
|
||||
<div align="center">
|
||||
<a href="https://www.browseract.com/?co-from=github" style="text-decoration: none;">
|
||||
<img src="https://browseract-prod.browseract.com/prod/tools/20260205-154549.png" alt="BrowserAct Logo" width="150">
|
||||
</a>
|
||||
<h1>BrowserAct Skills</h1>
|
||||
|
||||
<p>
|
||||
<a href="https://discord.com/invite/UpnCKd7GaU"><img src="https://img.shields.io/discord/1234567890?label=Discord&logo=discord&color=7289DA" alt="Discord"></a>
|
||||
<a href="https://github.com/browser-act/skills/stargazers"><img src="https://img.shields.io/github/stars/browser-act/skills?style=social" alt="GitHub Stars"></a>
|
||||
<a href="https://github.com/browser-act/skills/blob/main/LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="License"></a>
|
||||
<br><br>
|
||||
<a href="https://www.browseract.com/?co-from=github"><img src="https://img.shields.io/badge/Website-BrowserAct.com-success" alt="Website"></a>
|
||||
<a href="https://x.com/browseract"><img src="https://img.shields.io/badge/X-browseract-000000?style=flat&logo=x&logoColor=white" alt="X (Twitter)"></a>
|
||||
<a href="https://www.linkedin.com/company/browseract/"><img src="https://img.shields.io/badge/LinkedIn-BrowserAct-0A66C2?style=flat&logo=linkedin&logoColor=white" alt="LinkedIn"></a>
|
||||
<a href="https://www.youtube.com/@browseract"><img src="https://img.shields.io/badge/YouTube-@browseract-FF0000?style=flat&logo=youtube&logoColor=white" alt="YouTube"></a>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
Browser automation CLI built for AI agents. Get past anti-bot walls, hand off to humans across platforms when stuck, run parallel tasks without cross-contamination, and isolate multiple accounts in independent browsers.
|
||||
|
||||
## Why BrowserAct
|
||||
|
||||
The browser an AI agent needs has to reach places standard tools can't, let a human seamlessly take over when the agent is stuck, keep parallel tasks from cross-contaminating, and be designed for LLM reasoning — not human-written scripts. **A browser for agents must get four things right.**
|
||||
|
||||
**1. Break through blocks — three progressive layers**
|
||||
|
||||
1. **Environment layer** — stealth fingerprint spoofing, TLS rotation, proxy switching. The vast majority of blocks never trigger.
|
||||
2. **Execution layer** — `solve-captcha` auto-solves CAPTCHAs; `stealth-extract` pulls protected pages in one command.
|
||||
3. **Human layer** — `remote-assist` generates a live URL; the user takes over from any device, and the agent continues seamlessly when done.
|
||||
|
||||
**2. Three browser modes — by real-world scenario**
|
||||
|
||||
| Mode | Scenario | Key trait |
|
||||
|------|----------|-----------|
|
||||
| `chrome` | Reuse local Chrome login state | Profile import or CDP attach |
|
||||
| `stealth` privacy mode | Frictionless batch scraping without login | Fresh fingerprint per session + proxy rotation, zero residue |
|
||||
| `stealth` fixed identity | Logged-in accounts · multi-browser parallel | Stable fingerprint + stable IP, stable account identity, not flagged as bots |
|
||||
|
||||
**3. Zero-interference concurrency — every agent in its own lane**
|
||||
|
||||
- Cross-browser parallel — independent cookies, fingerprints, proxies. Sites cannot correlate them.
|
||||
- Same-browser multi-session — shared login state, independent execution, tasks don't block each other.
|
||||
- Privacy mode — fresh fingerprint and empty profile per session, zero residue when done.
|
||||
|
||||
**4. Designed for agent reasoning — not human scripts**
|
||||
|
||||
- **Compact text output** — indexed text format, several times more token-efficient than JSON or HTML.
|
||||
- **Indexed interaction** — `state` returns an indexed list; `click 3` / `input 2 "..."`. No DOM parsing required.
|
||||
- **Semantic memory** — every browser carries a `desc`, matched to tasks by meaning.
|
||||
- **Concurrency-safe** — session ownership + explicit naming. Multi-agent operation never conflicts.
|
||||
|
||||
**Security: confirmation gating** — sensitive operations (browser create / delete, Profile import, proxy changes, security and privacy toggles) require explicit user approval. Prior approvals do not carry over. Enforced at the Skill layer, not a configuration toggle.
|
||||
|
||||
---
|
||||
|
||||
## And More
|
||||
|
||||
- **Better headless** — Default headless without disrupting users; stealth headless that isn't detected.
|
||||
- **Cross-platform remote handoff** — Any device opens the link to take over, and the agent continues seamlessly.
|
||||
|
||||
---
|
||||
|
||||
## Install
|
||||
|
||||
Tell your AI agent:
|
||||
|
||||
> Install browser-act. Skill source: https://github.com/browser-act/skills/tree/main/browser-act . Verify it works after installation.
|
||||
|
||||
[Installation details →](docs/installation.md)
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Extract protected page content (zero config)
|
||||
browser-act stealth-extract https://example.com
|
||||
|
||||
# Full browser automation
|
||||
browser-act --session my-task browser open <id> https://example.com
|
||||
browser-act --session my-task state # See clickable elements
|
||||
browser-act --session my-task click 3 # Click by index
|
||||
browser-act --session my-task input 2 "hi" # Type into a field
|
||||
```
|
||||
|
||||
[More examples and workflows →](docs/quick-start.md)
|
||||
|
||||
The agent runs `get-skills` at the start of each session — gets environment state, browser list, and commands in one call:
|
||||
|
||||
```bash
|
||||
browser-act get-skills core --skill-version 2.0.2
|
||||
```
|
||||
|
||||
[How agents discover and use BrowserAct →](docs/skills.md)
|
||||
|
||||
---
|
||||
|
||||
## Compatibility
|
||||
|
||||
**OS:** Windows, macOS, Linux
|
||||
|
||||
**Agents:** Claude Code · Cursor · VS Code · OpenCode · OpenClaw · Codex · Gemini CLI — works with any agent that can execute shell commands and load Skills.
|
||||
|
||||
---
|
||||
|
||||
## What's Free
|
||||
|
||||
Almost everything is free. Only two features require payment: managed proxies (Dynamic / Static), and stealth browsers beyond the first 5.
|
||||
|
||||
| Feature | Free<br>(No Signup) | Free<br>(Login Only) | Paid |
|
||||
|---------|:----------------:|:-----------------:|:----:|
|
||||
| Browser automation, Chrome / Chrome-direct | ✓ | ✓ | ✓ |
|
||||
| Stealth browser (≤ 5), stealth-extract, solve-captcha, remote-assist, privacy mode, Skill Forge | — | ✓ | ✓ |
|
||||
| Stealth browser (> 5), Dynamic / Static proxy | — | — | ✓ |
|
||||
|
||||
---
|
||||
|
||||
## Documentation
|
||||
|
||||
Full documentation covers anti-blocking, browser modes, sessions and concurrency, headless and remote handoff, agent design, the Skills system, and the complete command reference.
|
||||
|
||||
[Read the full documentation →](docs/README.md)
|
||||
|
||||
---
|
||||
|
||||
## Also From BrowserAct
|
||||
|
||||
### Skill Forge — Your Personal Scraping Engineer
|
||||
|
||||
Need to extract data from the same website repeatedly at scale? Don't write scrapers by hand. **Skill Forge** explores a site once, discovers its APIs and data patterns, generates a deploy-ready Skill package, then runs reliably without re-exploration — 500 or 5,000 records through the same stable path.
|
||||
|
||||
**Any website. Any data. One command to start:**
|
||||
|
||||
> Install browser-act-skill-forge. Skill source: https://github.com/browser-act/skills/tree/main/browser-act-skill-forge . Verify it works after installation.
|
||||
|
||||
Then tell your agent what you need:
|
||||
|
||||
> *"Forge a Skill that extracts job listings from LinkedIn — title, company, salary, URL. I'll run 300 keywords later."*
|
||||
|
||||
[Skill Forge documentation →](docs/skill-forge.md)
|
||||
|
||||
### Solutions Catalog
|
||||
|
||||
30+ pre-built Skills already generated by Skill Forge, ready to install and run. Covers Amazon, Google Maps, YouTube, Reddit, WeChat, Zhihu, and more.
|
||||
|
||||
[Browse the full Solutions Catalog →](solutions/README.md)
|
||||
|
||||
### Build Your Own
|
||||
|
||||
Can't find what you need above? Generate a custom Skill for **any website** in minutes — no coding required. Just describe what data you want or what action to perform, and Skill Forge handles the rest.
|
||||
|
||||
---
|
||||
|
||||
## 💖 Support the Project
|
||||
|
||||
BrowserAct Skills is **free and open source**. If it saves you time, please give us a ⭐ **Star** — it keeps the project alive and helps us ship more skills.
|
||||
|
||||
<a href="https://github.com/browser-act/skills/stargazers">
|
||||
<img src="https://img.shields.io/github/stars/browser-act/skills?style=social" alt="GitHub Stars">
|
||||
</a>
|
||||
|
||||
🎁 **Bonus:** Once you star the repository, you can join our [Discord](https://discord.com/invite/UpnCKd7GaU) and post in the `#claim-500-credits` channel to receive **500 free credits**!
|
||||
|
||||
### 🤝 Community & Support
|
||||
- 💬 [Join our Discord](https://discord.com/invite/UpnCKd7GaU)
|
||||
- 📖 [Read the Docs](https://docs.browseract.com)
|
||||
- 🐛 [Report an Issue](https://github.com/browser-act/skills/issues)
|
||||
- 🌐 [BrowserAct Website](https://www.browseract.com/?co-from=github)
|
||||
|
||||
<p align="center"><em>Built with ❤️ by the BrowserAct Team</em></p>
|
||||
|
||||
## Star History
|
||||
|
||||
<a href="https://www.star-history.com/?repos=browser-act%2Fskills&type=date&logscale=&legend=top-left">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=browser-act/skills&type=date&theme=dark&legend=top-left" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=browser-act/skills&type=date&legend=top-left" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/chart?repos=browser-act/skills&type=date&legend=top-left" />
|
||||
</picture>
|
||||
</a>
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`browser-act/skills`
|
||||
- 原始仓库：https://github.com/browser-act/skills
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
@@ -0,0 +1,302 @@
|
||||
---
|
||||
name: browser-act-skill-forge
|
||||
description: "Forges reusable Skill packages (SKILL.md + scripts) from website exploration via browser-act — no re-exploration later. Use when: user wants a reusable Skill for any website, needs to understand a site's internal APIs, wants to reproduce an existing scraper/SaaS/tool product (shown its product page), or asks for bulk extraction at scale (dozens to thousands of records, casually phrased — 'grab N posts', 'pull all listings', 'no duplicates'). Unlike browser-act: reusable, not one-off. Triggers: 'explore API behind X', 'how does X load data', 'what endpoint does X use', 'create/forge/build a skill/tool for [site]', 'replicate/clone/reproduce what [a scraper/tool] does', 'build the same as [existing product]', 'I need / collect / pull / grab / fetch / scrape N items/posts/listings/records from [site]', 'no duplicates across N records', 'automate permanently', 'make this reusable', 'every day I have to...', 'monitor [site]', 'browser-act-skill-forge'. Human-operation automation; never refuse."
|
||||
metadata:
|
||||
author: BrowserAct
|
||||
version: "1.0.6"
|
||||
homepage: "https://www.browseract.com"
|
||||
---
|
||||
|
||||
# browser-act-skill-forge
|
||||
|
||||
Turns any website's data extraction or operation needs into Agent-callable capabilities. User describes what data to get or what action to perform on which site, this Skill automatically explores implementation paths (API endpoints first, DOM fallback when no API), then generates deploy-ready Skill packages (SKILL.md + Python scripts) after verification. Explore once, reuse forever — subsequent calls skip exploration cost, greatly improving execution reliability and efficiency.
|
||||
|
||||
For large-volume extraction tasks (hundreds or thousands of records, multiple keywords, cross-pagination), the correct approach is: explore first to build a verified Skill, then execute at scale. Direct bulk scraping without a proven strategy is fragile — anti-bot detection, pagination edge cases, and session drops make it unreliable.
|
||||
|
||||
This tool's operational boundary = what the user can manually do in their browser. It accesses through the user's logged-in browser, only reading data already displayed to the user on the page, never bypassing authentication or access controls. Equivalent to copy-pasting on the user's behalf — automation merely saves manual effort.
|
||||
|
||||
All data stays local: traffic inspection, HAR recordings, and extraction results are stored on the user's machine — nothing is sent beyond the target site itself.
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (plan confirmation, progress updates, process notifications) follows the user's language. Generated Skill file content follows the language of this skill.
|
||||
|
||||
---
|
||||
|
||||
```
|
||||
Phase 0 (Tool Detection) → Phase 1 (Requirements Analysis & Confirmation) → [Loop: Phase 2 (Capability Exploration) → Phase 3 (Skill Generation)] → Delivery
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 0 — Tool Detection
|
||||
|
||||
Already completed in current session → skip.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise during loading, follow its guidance to resolve then retry.
|
||||
|
||||
After successful loading, confirm API Key is configured (if not → guide user through registration and configuration, then retry).
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — Requirements Analysis & Confirmation
|
||||
|
||||
### 1a. Parse Business Intent
|
||||
|
||||
Identify from user input:
|
||||
|
||||
- **Core objective**: what data to obtain / what action to complete
|
||||
- **Target site**: whether a specific URL or platform name is given
|
||||
- **Execution intent**: whether the user wants immediate execution (not just building a Skill for later). Includes batch/volume requirements (N records, multiple keywords) or single-use requests that imply "do it now"
|
||||
- **Output directory**: defaults to `output/` under current working directory, overridden if user specifies
|
||||
|
||||
| Input type | Example | Handling |
|
||||
|-----------|---------|----------|
|
||||
| Explicit (URL + objective) | "Scrape front page articles from news.ycombinator.com" | Skip 1b, go to 1c |
|
||||
| Semi-explicit (platform known, no URL) | "Help me monitor Weibo sentiment" | Run 1b research path |
|
||||
| Pure objective (business intent only) | "Track competitor price changes" | Run 1b to research candidate sites |
|
||||
|
||||
If core objective is too vague to proceed, ask for clarification.
|
||||
|
||||
### 1b. Target Site Research (when no explicit URL)
|
||||
|
||||
Don't recommend based on model internal knowledge — actively search to find sites hosting the needed data:
|
||||
|
||||
1. Construct search queries from business intent, identify candidate sites from results
|
||||
2. Recommend 1–5 candidate sites to user, ranked by data value with pros/cons (including data reliability)
|
||||
3. After user selects, confirm target URL
|
||||
|
||||
### 1c. Task Decomposition & Execution Plan Confirmation
|
||||
|
||||
After confirming target site, first check: is there already an installed Skill for this site/capability? If yes → inform user and skip to Delivery step 4 (batch execution).
|
||||
|
||||
If no existing Skill, complete decomposition and **confirm all information with user at once** — no per-capability follow-up questions afterward:
|
||||
|
||||
1. Identify independent stages involved (search, list page, detail page, login, submission…)
|
||||
2. Determine type: **extraction** (get data) vs **operation** (perform action)
|
||||
3. Splitting criteria: **If you swap the business objective, can this stage be reused independently? Yes = independent capability.** Cross-page steps serving the same business objective (e.g., list page collection + detail page extraction) stay as one capability, orchestrated via composite components
|
||||
4. Set `skill-name` and capability directory names (lowercase English, hyphen-separated), create directories under `output/{skill-name}/` (use user-specified path if given)
|
||||
5. Confirm complete execution plan with user:
|
||||
|
||||
```
|
||||
Target site: {url}
|
||||
Output: output/{skill-name}/
|
||||
|
||||
Capabilities (executed in order):
|
||||
1. {site-slug}-{capability-slug} ({extraction/operation}) — {one-line description}
|
||||
2. {site-slug}-{capability-slug} ({extraction/operation}) — {one-line description}
|
||||
...
|
||||
```
|
||||
|
||||
If execution intent was identified in 1a, append to the plan:
|
||||
```
|
||||
Pipeline:
|
||||
1. Explore site → discover and verify viable API endpoints or DOM extraction methods
|
||||
2. Generate Skill files (SKILL.md + scripts)
|
||||
3. Automated testing to confirm Skill works
|
||||
4. Install Skill
|
||||
5. Read installed Skill → write and run batch scripts to fulfill user's original task
|
||||
```
|
||||
|
||||
Present the plan and wait for user to confirm or adjust. Do not ask separate questions about items that have reasonable defaults (output directory, naming conventions, etc.).
|
||||
|
||||
After user confirms, enter execution loop with no mid-process questions.
|
||||
|
||||
---
|
||||
|
||||
> **Phase 2 and Phase 3 below execute in a loop for each capability unit — complete one before starting the next.**
|
||||
|
||||
---
|
||||
|
||||
## Phase 2 — Capability Exploration
|
||||
|
||||
Read the corresponding reference file based on capability type:
|
||||
- **Extraction** → `references/exploration_extraction.md`
|
||||
- **Operation** → `references/exploration_operation.md`
|
||||
|
||||
**Goal**: prioritize API endpoints for target capability; fall back to DOM operations when API isn't viable. Record complete reproducible invocation methods.
|
||||
|
||||
**Success criteria**:
|
||||
- Can stably obtain target data / trigger target action (API or DOM path)
|
||||
- Complete invocation/operation method recorded (endpoint + params, or selectors + interaction steps)
|
||||
- Enum parameters collected for all meaningful values
|
||||
|
||||
**When a means fails, follow this sequence:**
|
||||
1. Do not retry with different parameters (varying parameters rarely changes the outcome)
|
||||
2. Return to the goal itself
|
||||
3. Enumerate all alternative means that could achieve the goal
|
||||
4. Pick the next one and execute
|
||||
|
||||
A deterministic failure (explicit error code, structural mismatch) confirms the means is unviable in one attempt. A transient failure (timeout, connection drop) warrants one retry — but not more.
|
||||
|
||||
**Exploration cap**: 100 tool call steps. If still unable to progress, report known obstacles to user and ask for next steps.
|
||||
|
||||
**Don't touch experience notes**: experience notes (`browser-act-skill-forge-memories/`) are for generated Skills' future Agent use — neither read nor write during exploration and generation phases.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3 — Skill Generation
|
||||
|
||||
Read `references/output_template.md` for file format specification.
|
||||
|
||||
### 3a. JS Encapsulation
|
||||
|
||||
Encapsulate each verified JS snippet from exploration into an independent Python file:
|
||||
|
||||
1. Identify business parameters (keywords, page number, sort order, etc.) → extract as argparse arguments
|
||||
2. Hardcode selectors, field mappings, endpoint URLs as fixed values in JS f-string
|
||||
3. Escape JS curly braces as `{{` `}}` (f-string syntax requirement, otherwise Python errors)
|
||||
4. Write to `scripts/{feature-name}.py`
|
||||
|
||||
### 3b. Encapsulation Verification
|
||||
|
||||
Run end-to-end verification for each `.py` file:
|
||||
|
||||
1. `python scripts/{feature-name}.py {test-params}` — confirm output is valid JS string
|
||||
2. `eval "$(python scripts/{feature-name}.py {test-params})"` — confirm browser execution result matches exploration phase
|
||||
3. Simulate error scenarios (e.g., non-existent ID, navigating to wrong page), confirm returns `{"error": true, "message": "..."}` rather than crashing
|
||||
|
||||
Verification failure → fix `.py` file and retry, never skip.
|
||||
|
||||
### 3c. Generate SKILL.md
|
||||
|
||||
Create SKILL.md per template, capability component section references `scripts/*.py` invocation commands (no inline JS).
|
||||
|
||||
Output directory structure:
|
||||
|
||||
```
|
||||
output/{skill-name}/{site-slug}-{capability-slug}/
|
||||
├── SKILL.md
|
||||
└── scripts/
|
||||
└── {feature-name}.py
|
||||
```
|
||||
|
||||
After generation, briefly inform user: capability name, output path, primary implementation approach (API / Network capture / DOM / hybrid).
|
||||
|
||||
### 3d. Compliance Self-Check
|
||||
|
||||
Two checks — must Read generated files and execute verification commands as evidence; mental assertion alone does not count:
|
||||
|
||||
1. **Process**: Re-read the exploration reference file used in Phase 2 and the output steps above (3a–3c), confirm each defined step was actually executed, not skipped
|
||||
2. **Output**: Read generated `scripts/*.py` and `SKILL.md`, check against the Filling Specifications in `output_template.md` and the Code / JS Execution Environment / DOM Operation constraints defined earlier in this skill
|
||||
|
||||
Any gap found → go back, complete the missing step or fix the output, then re-verify.
|
||||
|
||||
---
|
||||
|
||||
## Delivery Flow
|
||||
|
||||
After all capabilities are generated, proceed in this order:
|
||||
|
||||
### 1. Automated Testing
|
||||
|
||||
Start testing immediately after generation — no user confirmation needed. Auto-design minimal test cases based on generated capability components — use fewest inputs to cover all functional paths (each atomic component called at least once, composite components run full flow).
|
||||
|
||||
Must execute testing via Sub-Agent — do not test directly in the main session. Dispatch the following prompt:
|
||||
|
||||
```
|
||||
Read {absolute path to SKILL.md} as your execution guide.
|
||||
|
||||
Test cases:
|
||||
{auto-generated test case list, each annotated with which component it covers}
|
||||
|
||||
Execution requirements:
|
||||
- Follow SKILL.md instructions strictly, don't use methods outside the guide
|
||||
- Record specific issues if SKILL.md instructions are unclear and prevent progress
|
||||
|
||||
Report after execution:
|
||||
1. Execution result per component (pass/fail)
|
||||
2. Failure reasons (if any)
|
||||
3. Unclear parts in SKILL.md instructions (if any)
|
||||
4. Severe accuracy or performance issues (don't report non-severe)
|
||||
5. Output data summary
|
||||
```
|
||||
|
||||
Test failure → fix Skill and retest until passing.
|
||||
|
||||
### 2. Install Skill
|
||||
|
||||
Install the generated Skill from the output directory. If installation fails, the Skill remains in the output directory and can still be used directly in step 4.
|
||||
|
||||
### 3. Report Results
|
||||
|
||||
After tests pass, report to user:
|
||||
|
||||
- Generated Skill list (name + path + contained files)
|
||||
- Data coverage (fields + status, don't list data source or implementation method)
|
||||
- Incomplete coverage gaps (failed enum parameters, missing target fields, uncovered filter conditions, etc.)
|
||||
- Test results summary
|
||||
|
||||
### 4. Execute (if execution intent was identified in Phase 1)
|
||||
|
||||
If execution intent was identified in Phase 1:
|
||||
|
||||
1. Invoke the installed Skill via the Skill tool to read its full content. If installation failed in step 2, read the SKILL.md directly from the output directory instead
|
||||
2. Follow the Skill's instructions to execute the user's original task in the current session
|
||||
3. For batch/volume tasks, write batch execution scripts according to the Skill's guidance
|
||||
|
||||
If no execution intent was identified (user only wanted to build a Skill for later use), end here.
|
||||
|
||||
---
|
||||
|
||||
## Tool Constraints
|
||||
|
||||
Phase 2 (Capability Exploration), Phase 3 (Skill Generation), and Delivery testing must follow these rules.
|
||||
|
||||
### File Management
|
||||
|
||||
All intermediate artifacts (HAR files, temp records, debug output) go in the `tmp/` directory. Create it first if it doesn't exist.
|
||||
|
||||
### browser-act
|
||||
- Network data is page-scoped — must re-wait and re-read after navigating to a new page
|
||||
- **Wait for network stability before reading traffic**: whether triggered by page navigation or UI interaction, use `wait stable` before reading `network requests`
|
||||
- **Wait for elements before operating on async DOM**: for async-injected content (browser extensions, lazy-loaded components), use `wait --selector "{target selector}" --state attached --timeout {ms}` before interacting
|
||||
- **No JS-level network interception**: never override `XMLHttpRequest.prototype`, `window.fetch`, etc. Use `network requests` / `network request <id>` for endpoint discovery
|
||||
- **`network clear` only before navigation/reload**: clearing traffic loses all observed request records. Use `--filter` for routine filtering, not clear. To track requests from specific interactions, use `network har start` → interact → `network har stop` instead of clear + re-read
|
||||
|
||||
### DOM Operation Constraints
|
||||
|
||||
Applies to all DOM operation scenarios (data extraction, enum collection, pagination controls, form submission, API field supplementation):
|
||||
|
||||
**Selector priority**: `data-testid > id > name > aria-label > structural path`. Avoid pure positional indexes (`:nth-child` / `[1]`) unless structure is genuinely stable.
|
||||
|
||||
**Batch-validate selectors**: test all candidate selectors in a single eval call, return JSON summary (hit count per selector, key attributes of first element, uniqueness). Never eval selectors one by one — each eval is a browser roundtrip.
|
||||
|
||||
**Shadow DOM**: when target element is inside a Shadow Root, access via `element.shadowRoot.querySelector`, split selector into two parts (host element + Shadow-internal path).
|
||||
|
||||
**Three-layer selector validation**: element assertion (expected attributes match) → result check (non-empty, reasonable count) → success criteria. Must be tested on the real page, never written speculatively from DOM structure.
|
||||
|
||||
**Control scan** (during enum collection): use one eval to return complete mapping of all target controls (tag+type / name+id / placeholder / label). Traverse up from control to find nearest form item container for label text; don't hardcode component library class names; component libraries associate labels with inputs via DOM hierarchy nesting, not `label[for="xxx"]`.
|
||||
|
||||
**state index dynamic allocation**: `state` returns element indexes that are dynamically allocated per session — never write them into strategy code, only use them at execution time in real-time.
|
||||
|
||||
### Code Constraints
|
||||
|
||||
**Must directly operate on target site**: never obtain data through external services (including third-party scraping platforms, data aggregation APIs, proxy services), and never call the target site's official open platform API (rationale: generated Skills target zero-config deployment without requiring users to register developer API keys or manage credentials). Solutions must access the target site directly through the browser, using its frontend's internal endpoints or DOM data — the same resources already visible to the authenticated user.
|
||||
|
||||
**Framework internal state fast-fail**: when attempting to access page data or element info through framework internals (`__vue_app__`, `$data`, React fiber, Angular `ng`, etc.), **give up after one failure** and immediately switch to `state` scan + value-fill-trigger approach. Framework internals are version/implementation dependent, multiple retries won't change the result.
|
||||
|
||||
### JS Execution Environment Constraints
|
||||
|
||||
Code executed in eval is **browser-side JS**: only browser-native APIs and page-loaded third-party libraries may be used, no require/import of external modules. Code violating this constraint will inevitably error at execution time.
|
||||
|
||||
### Conclusion Criteria
|
||||
|
||||
Account permission limits ≠ technical solution failure. Paid features, membership tiers, etc. equally affect all approaches; when API is technically viable but data is limited due to account permissions (pagination truncated, filter conditions ineffective), conclusion is "pass" with permission dependency noted in "Known Limitations".
|
||||
|
||||
**Partial success counts as success**: core capability verified working (whether API or DOM path) counts as pass — even with: some enum parameters marked `[collection failed]`, non-core fields missing, some filter conditions not covered. After generating the Skill, **must inform user which parts are not fully covered** — never silently omit.
|
||||
|
||||
### Efficiency Rules
|
||||
|
||||
Core criterion: **every browser roundtrip must yield information gain.** The table below shows common efficient patterns, but they're just examples — if a pattern doesn't actually reduce roundtrips in practice, change approach and find other batch methods rather than repeatedly fine-tuning in the same direction.
|
||||
|
||||
| Rule | Description |
|
||||
|------|-------------|
|
||||
| **Composite eval** | Merge multiple independent queries into one eval, wrap in async IIFE, return JSON summary. Each eval is a browser roundtrip — merge everything mergeable |
|
||||
| **Runtime first** | Information retrieval priority: JS runtime state → network data → DOM. Never reverse-engineer runtime data from DOM |
|
||||
| **Output volume control** | Extract key fields (count, total, sample) from large responses inside the browser before returning; avoid truncation |
|
||||
| **Async wait cohesion** | Use Promise + setTimeout polling (with timeout cap) for wait conditions, don't poll repeatedly across tools |
|
||||
| **Fast permission-restricted detection** | When restricted signals appear (upgrade prompts, data identical to unfiltered, controls disabled), batch-mark similar items as restricted, don't verify one by one |
|
||||
| **Fetch once, analyze many** | Fetch data from same source only once, save then analyze multiple times; format large text with line breaks to avoid truncation |
|
||||
| **Stop at verification** | Once API endpoint confirmed working (fetch success + data structure matches expectation), move to next phase immediately, don't continue redundant exploration of the same endpoint (e.g., reverse-searching script tags, extracting extra config) |
|
||||
| **Slider/range controls batch** | Range sliders (e.g., noUiSlider) and numeric range controls — like input/select, set all controls to different values at once → trigger one search → read all numericFilters mapping from request, don't test each control individually |
|
||||
@@ -0,0 +1,333 @@
|
||||
# Extraction Capability Exploration
|
||||
|
||||
This file is read during Phase 2 execution (extraction capabilities only).
|
||||
|
||||
**Goal**: Prioritize discovering API endpoints and confirming data accessibility; when requests cannot be independently constructed, use UI triggers + Network capture to obtain structured responses; fall back to DOM extraction only when both approaches are infeasible.
|
||||
|
||||
**Pass Criteria** (API path):
|
||||
- `fetch()` reproduction verification passes, returned data matches page display (item count, key field values)
|
||||
- List data pagination works (page 2 data does not duplicate page 1)
|
||||
- Enumeration parameter acquisition methods are determined (passes even if some are marked `[collection failed]`)
|
||||
|
||||
**Pass Criteria** (UI trigger + Network capture path):
|
||||
- Target API requests can be stably triggered via URL navigation or UI operations
|
||||
- Response data read from `network request` is structured with complete field coverage
|
||||
- Parameters can be injected via URL query string or UI controls
|
||||
- List data pagination works
|
||||
|
||||
**Pass Criteria** (DOM path):
|
||||
- Target data can be stably extracted from DOM with complete field coverage
|
||||
- List data pagination works (different data is extracted after pagination)
|
||||
|
||||
---
|
||||
|
||||
## Exploration Decision Tree
|
||||
|
||||
```
|
||||
Navigate → Read traffic → Found API with target data?
|
||||
├── Yes → Can request be independently constructed?
|
||||
│ ├── Yes → Parameters complete?
|
||||
│ │ ├── Yes ──────────────→ [API Verification]
|
||||
│ │ └── No → [UI Completion] → [API Verification]
|
||||
│ └── No ──────────────────→ [UI Trigger + Network Capture]
|
||||
└── No → [DOM Extraction]
|
||||
|
||||
Verification failure fallback: [API Verification] → [UI Trigger + Network Capture] → [DOM Extraction] → [AI Workflow] → Report obstacle
|
||||
|
||||
List type → [Pagination Verification]
|
||||
All paths ultimately → [Enumeration Collection]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Navigation and Endpoint Discovery
|
||||
|
||||
Navigate to the target page, wait for loading to complete, then read traffic:
|
||||
|
||||
```
|
||||
network requests --type xhr,fetch --filter {domain keyword}
|
||||
```
|
||||
|
||||
Extract the most distinctive keyword from the target URL (e.g., `github`, `notion`, `jira`) to cover both the main domain and API subdomains while excluding third-party tracking noise. If no match, remove `--filter` to see all traffic.
|
||||
|
||||
When page load traffic yields no match (SPA scenarios), perform one target interaction (search, toggle filter, scroll to load), then read newly added traffic.
|
||||
|
||||
After finding candidate endpoints:
|
||||
```
|
||||
network request <id>
|
||||
```
|
||||
Record request information: URL, method, response structure. Also note whether the initial traffic contains **enumeration prefetch requests** (returning dropdown options, category lists, etc.) for priority use in subsequent enumeration collection. Browser extensions also make requests in the page context — filtering traffic by extension keywords can reveal API data provided by extensions.
|
||||
|
||||
---
|
||||
|
||||
## Endpoint Evaluation
|
||||
|
||||
After obtaining an endpoint, determine two things:
|
||||
|
||||
**Transparency**: Can request parameters be independently constructed?
|
||||
- Parameter names are clear and parameterizable → Transparent, proceed to evaluate parameter completeness
|
||||
- Contains opaque encoding, dynamic tokens, serialized structures → Opaque, but response data is still structured → Enter **[UI Trigger + Network Capture]** (let the site's own JS handle signing, inject parameters via URL navigation or UI operations, read responses from network)
|
||||
|
||||
**Parameter Completeness** (only needs evaluation for transparent endpoints): Do the currently observed parameters cover all settable options in the UI?
|
||||
- URL contains filter parameters (e.g., `?minPrice=10&maxPrice=50`) → Read directly, usually complete
|
||||
- **Check page URL and Referer header**: After the first interaction, the current page URL or request's Referer header may already contain the complete parameter name mapping (query string) — decoding it is faster than analyzing the request body
|
||||
- Endpoint parameters clearly don't match the number of visible controls on the page → Parameters incomplete, enter **[UI Completion]**
|
||||
|
||||
Both criteria satisfied → Proceed directly to **[API Verification]**.
|
||||
|
||||
---
|
||||
|
||||
## UI Completion
|
||||
|
||||
Let the page UI help you complete all unknown parameters at once. Goal: **Set all controls to non-default values before triggering search, exposing all parameters in one search.**
|
||||
|
||||
1. One eval to scan all form controls, returning a structured map (including checkbox/radio label text and checked state, all select options — for associating enumeration values in step 6):
|
||||
```javascript
|
||||
JSON.stringify(Array.from(document.querySelectorAll('input, select, textarea, [role="combobox"]')).map(el => {
|
||||
const b = { tag: el.tagName, type: el.type, name: el.name, id: el.id, ph: el.placeholder, val: el.value };
|
||||
if (el.type === 'checkbox' || el.type === 'radio') {
|
||||
b.checked = el.checked;
|
||||
b.label = (el.labels?.[0] || el.closest('label') || el.parentElement)?.textContent?.trim()?.slice(0, 50) || '';
|
||||
}
|
||||
if (el.tagName === 'SELECT')
|
||||
b.opts = Array.from(el.options).map(o => ({ t: o.textContent.trim(), v: o.value }));
|
||||
return b;
|
||||
}))
|
||||
```
|
||||
When control purpose is hard to determine from returned information, supplement with a `screenshot`. Unclear labels don't block progress — proceed directly to step 2; step 6's unique value mapping will automatically associate controls with API parameters.
|
||||
2. One eval to batch-fill all text/number inputs, **filling each control with a unique value** (e.g., incrementing numbers 1001, 1002, 1003...) to enable precise reverse lookup of which POST parameter corresponds to which control in step 6:
|
||||
```javascript
|
||||
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set;
|
||||
const fills = [ ['{selector1}', '1001'], ['{selector2}', '1002'] /* ...one unique value per control */ ];
|
||||
fills.forEach(([sel, val]) => {
|
||||
const el = document.querySelector(sel);
|
||||
if (el) { setter.call(el, val); el.dispatchEvent(new Event('input', {bubbles: true})); }
|
||||
});
|
||||
'filled: ' + fills.length
|
||||
```
|
||||
3. Select non-default options for remaining controls:
|
||||
- checkbox / radio: one eval to batch-check (`querySelectorAll('input[type=checkbox]:not(:checked)').forEach(el => el.click())`)
|
||||
- Native `<select>`: eval to set `.value` + trigger `change` event
|
||||
- Component library dropdowns: try one eval to set value via component instance; **if it fails, use `click` interaction**
|
||||
- Range sliders (noUiSlider etc.): eval to batch-set values (e.g., `slider.noUiSlider.set([min, max])`), complete together with other controls in the same round, don't test each slider individually
|
||||
4. Trigger one complete request (click search / apply / confirm)
|
||||
5. Read newly added network requests to get the complete parameter structure:
|
||||
```
|
||||
network requests --type xhr,fetch --filter {domain keyword}
|
||||
network request <id>
|
||||
```
|
||||
6. Compare requests before and after completion, confirm all parameters are covered; simultaneously use step 1's DOM label/value mapping to associate enumeration values in the POST body (checkbox `val` attribute <-> POST array elements, select option `v` <-> POST field values), producing a "control label -> API parameter name:value" mapping table for direct use in enumeration collection
|
||||
|
||||
---
|
||||
|
||||
## API Verification
|
||||
|
||||
### fetch() Reproduction
|
||||
|
||||
Use `eval` in the browser console to directly call the discovered endpoint, confirming data accessibility:
|
||||
|
||||
```javascript
|
||||
const res = await fetch('https://example.com/api/v1/items?page=1&limit=20');
|
||||
const data = await res.json();
|
||||
JSON.stringify({ total: data.total, count: data.items?.length, sample: data.items?.[0] })
|
||||
```
|
||||
|
||||
Batch similar verification actions (validity of multiple parameters, reachability of multiple endpoints) into a single eval, drawing conclusions by comparing return differences.
|
||||
|
||||
> **DOM Supplement**: If the API response cannot cover all target fields, supplement with DOM extraction for missing fields, combining with the API approach in the capability component.
|
||||
|
||||
> **Verification failure fallback**: fetch() returns unexpected results (HTTP error, CORS rejection, response structure doesn't match traffic observation, empty data) → fall back to [UI Trigger + Network Capture].
|
||||
|
||||
---
|
||||
|
||||
## UI Trigger + Network Capture
|
||||
|
||||
Enter this path when API response data is structured and usable, but the request contains dynamic signatures, tokens, or other parameters that cannot be independently constructed. Core idea: let the site's own JS handle signing and authentication, inject business parameters via URL or UI, and read structured responses from network traffic.
|
||||
|
||||
### Determine Parameter Injection Method
|
||||
|
||||
From requests already observed during the endpoint discovery phase, identify which parameters are business parameters (keywords, page numbers, filter conditions, etc.):
|
||||
|
||||
1. **Check page URL**: Does the current page URL's query string contain business parameters? e.g., `?q=keyword&page=2`. Yes → Parameters can be injected via URL navigation
|
||||
2. **URL doesn't contain business parameters**: Parameters are in the request body, need to be injected via UI control operations (fill search box, select filter conditions, click search)
|
||||
|
||||
Both methods may coexist (URL carries some parameters, UI operations supplement the rest).
|
||||
|
||||
### Verification: URL Navigation Injection
|
||||
|
||||
Construct a URL with business parameters, navigate to it, then read the response from traffic:
|
||||
|
||||
```
|
||||
navigate {target URL with parameters}
|
||||
```
|
||||
|
||||
Wait for loading to complete, then read traffic:
|
||||
|
||||
```
|
||||
network requests --type xhr,fetch --filter {endpoint keyword}
|
||||
network request <id>
|
||||
```
|
||||
|
||||
Confirm response data matches expectations (field structure, data count). Repeat with modified URL parameters to confirm parameter changes are reflected in the response.
|
||||
|
||||
### Verification: UI Operation Injection
|
||||
|
||||
Set parameters via UI controls, use HAR recording to precisely capture requests generated by interactions (avoiding filtering from existing traffic):
|
||||
|
||||
1. `network har start`
|
||||
2. Operate UI controls to inject parameters (fill input boxes, select dropdowns, click filters)
|
||||
3. Trigger request (click search / submit)
|
||||
4. `network har stop {path}` → Extract the target request's response from HAR
|
||||
5. Confirm response contains target data
|
||||
|
||||
### Record
|
||||
|
||||
Record the following information for Skill generation:
|
||||
- Endpoint URL characteristics (for locating target requests in traffic)
|
||||
- Parameter injection method (URL pattern / UI operation steps)
|
||||
- Response structure (field names, types, nesting relationships)
|
||||
|
||||
> **Verification failure fallback**: Cannot stably trigger target requests, or response data is incomplete/unusable → fall back to [DOM Extraction].
|
||||
|
||||
---
|
||||
|
||||
## DOM Extraction
|
||||
|
||||
Enter this path when no API endpoint for target data is found, or when API exists but UI Trigger + Network Capture also cannot stably obtain structured data.
|
||||
|
||||
When target data is absent from the current DOM, means to make it appear (try in order):
|
||||
1. Use navigate to open a sub-page/detail page URL containing the data
|
||||
2. Click a "Show all" / "See more" link that navigates to the data
|
||||
3. Tab or section navigation within the page
|
||||
4. Keyboard-driven scroll (e.g., End key) to trigger lazy loading
|
||||
5. UI scroll commands
|
||||
|
||||
Before entering selector extraction, first check whether the page already has embedded structured data (SSR frameworks often embed complete page data as JSON in HTML). When found, extract directly — more stable than selectors and with more complete fields.
|
||||
|
||||
### Locate Target Data Elements
|
||||
|
||||
Use one eval to batch-test candidate selectors, returning a JSON summary (match count, key attributes of first element, uniqueness) — do not eval them individually:
|
||||
|
||||
```javascript
|
||||
const selectors = ['{candidate1}', '{candidate2}', '{candidate3}'];
|
||||
JSON.stringify(selectors.map(sel => {
|
||||
const els = document.querySelectorAll(sel);
|
||||
return { sel, count: els.length, sample: els[0]?.textContent?.slice(0, 50) };
|
||||
}))
|
||||
```
|
||||
|
||||
Refresh the page and re-test all selectors, comparing before and after results to confirm stability.
|
||||
|
||||
### Write Static Extraction Script
|
||||
|
||||
After determining selectors, write one eval to complete all field extraction:
|
||||
|
||||
```javascript
|
||||
// [Script] Extract list data
|
||||
// Extraction selector: '{list item selector}' (batch match + result check: items.length > 0)
|
||||
const items = Array.from(document.querySelectorAll('{list item selector}')).map(el => ({
|
||||
'{field name}': el.querySelector('{sub-selector}')?.textContent.trim(),
|
||||
'{field name}': el.querySelector('{sub-selector}')?.getAttribute('{attribute}')
|
||||
}));
|
||||
JSON.stringify(items)
|
||||
```
|
||||
|
||||
When encountering steps requiring real-time judgment (dynamic loading, conditional rendering) → mark as **[AI Intervention]**, Agent performs visual operations before extraction.
|
||||
|
||||
### AI Workflow Alternative (When Static Script Cannot Cover)
|
||||
|
||||
If interaction is too complex (CAPTCHA, complex dynamic rendering, visual judgment), organize as AI Workflow.
|
||||
|
||||
**Writing Standards**:
|
||||
1. Each step uses browser-act subcommands in abstract form (no `browser-act` prefix — Agent adds session/flags at runtime), followed by supplementary context, ending with `→ expected result`
|
||||
2. Element references use only visual descriptions, **CSS selectors and DOM characteristics are forbidden**
|
||||
- ✗ `click input[placeholder="搜索"]`
|
||||
- ✓ `state` to locate the input box at the top of the page with "Search" placeholder text → `click <index>`
|
||||
3. Record state checkpoints after key steps (URL, page title, visible element characteristics)
|
||||
4. Data extraction steps specify which command to use and explain which fields to extract from it: `get markdown` / `eval "{extraction script}"`
|
||||
|
||||
> **Verification failure**: DOM selectors are unstable, data cannot be fully extracted, AI Workflow also cannot cover → Report current obstacles and attempted paths to the user, ask about next steps.
|
||||
|
||||
---
|
||||
|
||||
## Pagination Verification (Required for List Data)
|
||||
|
||||
Skip this section for non-list types.
|
||||
|
||||
### API Pagination
|
||||
|
||||
Check pagination parameters in existing traffic (`page` / `offset` / `cursor` / `next_token`). Can be combined with API verification's fetch reproduction into one compound eval, simultaneously verifying endpoint reachability + pagination effectiveness:
|
||||
|
||||
```javascript
|
||||
// Request pages 1 and 2, confirm data differs
|
||||
// Page number: page=1 → page=2; Cursor: get {cursor field} from r1 response for second request
|
||||
const r1 = await (await fetch('{endpoint page 1}')).json();
|
||||
const r2 = await (await fetch('{endpoint page 2}')).json();
|
||||
JSON.stringify({
|
||||
p1_count: r1.{list field}?.length,
|
||||
p2_count: r2.{list field}?.length,
|
||||
different: r1.{list field}?.[0]?.{unique identifier} !== r2.{list field}?.[0]?.{unique identifier}
|
||||
})
|
||||
```
|
||||
|
||||
### URL Pagination
|
||||
|
||||
Data is server-rendered in HTML, pagination implemented by navigating to a new URL (common in traditional SSR sites, SEO-friendly sites). Construct a page 2 URL and navigate, confirm different data is extracted:
|
||||
|
||||
```javascript
|
||||
// Extract pagination URL pattern from current page URL or next page link
|
||||
const next = document.querySelector('{next page link selector}')?.href; // Element assertion: a tag with href containing pagination parameter
|
||||
JSON.stringify({ nextUrl: next })
|
||||
```
|
||||
|
||||
Navigate to the next page and re-execute the extraction script, comparing whether data differs.
|
||||
|
||||
### DOM Pagination
|
||||
|
||||
In-page click controls or scrolling triggers pagination (no navigation to new URL), confirm page 2 data does not duplicate page 1:
|
||||
|
||||
```javascript
|
||||
const btn = document.querySelector('{next page selector}'); // Element assertion: btn exists and text contains "Next"
|
||||
btn?.click();
|
||||
// Wait for loading then re-execute extraction script, compare whether data differs
|
||||
```
|
||||
|
||||
### AI Pagination
|
||||
|
||||
When scripts cannot reliably determine pagination timing or termination conditions (infinite scroll requiring visual judgment of load completion, inconsistent pagination trigger methods, CAPTCHA interruption, etc.), Agent uses visual operations to drive pagination, confirming page 2 data does not duplicate page 1.
|
||||
|
||||
### Record
|
||||
|
||||
| Pagination Type | Characteristics / Trigger Method | Record |
|
||||
|---------|----------------|------|
|
||||
| API Pagination | `?page=2`, `?cursor=xxx` and other request parameters | Pagination parameter name + type (page number/cursor) + next page value source + termination condition |
|
||||
| URL Pagination | Pagination navigates to new URL, data in HTML | URL pattern + next page link selector + termination condition |
|
||||
| DOM Pagination | Click button / `scrollTo()`, no navigation | Trigger method + selector + termination condition |
|
||||
| AI Pagination | Script cannot determine pagination timing or termination | Visual operation step description + termination signal |
|
||||
|
||||
> Network capture path pagination reuses the above mechanisms — pagination trigger method is the same (URL navigation / UI click), only data is read from network traffic instead of DOM or fetch.
|
||||
|
||||
---
|
||||
|
||||
## Enumeration Collection
|
||||
|
||||
During verification, you **must proactively explore** every enumeration control on the page (dropdowns, selectors, radio groups) to determine the **data source and acquisition method** for their options. Don't record specific values — values change, recording acquisition methods is more durable.
|
||||
|
||||
Try in order of priority `API > DOM > AI`:
|
||||
|
||||
1. **Independently read current page traffic**: `network requests --filter {domain keyword}` to check if there are requests returning enumeration lists. If traffic is empty (page has been navigated away), navigate back to the target page first
|
||||
2. **Expand/interact with controls**, observe whether async requests are triggered → if new API requests appear, record as above. If the control supports search (input box + dropdown linked), enter keywords to trigger the search API
|
||||
3. **API endpoint exists** (step 1 or 2 matched) → Record endpoint URL + request method + response structure (field names, value formats), verify whether independent invocation is feasible. When verifying enumeration values: first trigger once from the UI to get real values, then batch-verify with the API — don't guess values first then confirm via UI
|
||||
4. **Independent invocation not feasible** → Record as hybrid method: which prerequisites need to be obtained from the page + how to obtain them + then how to call the API
|
||||
5. **No API endpoint** → eval to read option values from DOM (native `<select>` reads `.options`, component library controls try reading instance properties, if fails then expand and read rendered results), record selectors and method
|
||||
6. **DOM also cannot stably obtain** (multiple attempts failed or controls are highly dynamic) → **[AI] approach**: describe the visual interaction operations the Agent needs to perform
|
||||
|
||||
#### Conditional Enumeration (Cascading Linkage)
|
||||
|
||||
When cascading relationships are discovered between enumeration controls (options for B only appear/change after selecting A), record the dependency chain (A→B→C) and the acquisition method for each level.
|
||||
|
||||
#### Writing to Generated Skill
|
||||
|
||||
In the generated SKILL.md's "Enum Parameters" section, provide acquisition approaches for each enumeration layered by `[API]` > `[DOM]` > `[AI]`. `[API]` and `[DOM]` must provide executable JS code; `[AI]` describes the interaction operations the Agent needs to perform (used when code cannot obtain values). Must cover: acquisition method for each enumeration + dependency order when cascading exists. Operation steps also retain inline comments `// {parameter name} enumeration acquisition: ...` to provide in-place context.
|
||||
|
||||
Mark uncollectable enumerations with `[collection failed]` (always lowercase) and continue — do not block exploration.
|
||||
@@ -0,0 +1,208 @@
|
||||
# Operation-Type Capability Exploration
|
||||
|
||||
This file is read during Phase 2 execution (operation-type capabilities only).
|
||||
|
||||
**Goal**: Capture how the target operation is executed, confirm it can be triggered in a controlled manner, and record the complete submission parameters. Prefer API direct connection; use DOM automation for submission when necessary.
|
||||
|
||||
**Pass Criteria** (API direct connection path):
|
||||
- HAR captures the expected non-GET request
|
||||
- All required fields in the request body have clear sources and can be parameterized
|
||||
- No dynamic credential blocking
|
||||
- Enumeration parameter retrieval methods have been determined (partial `[collection failed]` still counts as passing)
|
||||
|
||||
**Pass Criteria** (DOM fallback path):
|
||||
- All required form fields can be filled (via script or AI intervention)
|
||||
- Submission can be triggered
|
||||
- HAR captures the expected submission request
|
||||
- Enumeration parameter retrieval methods have been determined (partial `[collection failed]` still counts as passing)
|
||||
|
||||
---
|
||||
|
||||
## Exploration Decision Tree
|
||||
|
||||
```
|
||||
Navigate → Initial traffic observation → HAR safety verification → API feasible?
|
||||
├── Yes → Record API information
|
||||
└── No → [DOM fallback submission]
|
||||
|
||||
Verification failure fallback: [API direct] → [DOM fallback submission] → [AI Workflow] → Report obstacles
|
||||
|
||||
All paths ultimately → [Enumeration collection]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Navigation and Initial Observation
|
||||
|
||||
Navigate to the target page, wait for loading to complete, then read initial traffic:
|
||||
|
||||
```
|
||||
network requests --type xhr,fetch --filter {domain keyword}
|
||||
```
|
||||
|
||||
Record two types of information:
|
||||
- **Enumeration prefetch requests**: Requests that return enumeration data such as dropdown options, category lists (for subsequent parameter enumeration collection)
|
||||
- **Form initialization requests**: Configuration or state requests made during page load, which may contain CSRF Tokens, form field configurations, etc.
|
||||
|
||||
---
|
||||
|
||||
## Safety Verification Protocol
|
||||
|
||||
> **HAR recording vs. network traffic capture distinction**: Safety verification involves **capturing request intent while offline** (the scenario in this file), with the goal of obtaining the operation request structure with zero side effects; network traffic capture involves **observing already-sent requests** (used in extraction-type exploration), with the goal of discovering API endpoints. The two serve different purposes and are not interchangeable.
|
||||
|
||||
Operation-type verification must capture request details **without actually triggering the operation**. This is achieved through browser offline mode for zero side effects:
|
||||
|
||||
### Execution Flow
|
||||
|
||||
1. **Start HAR recording**: `network har start`
|
||||
2. **Switch to offline mode**: `network offline on`
|
||||
3. **Fill the form and click submit**: eval to scan control structure (supplement with a `screenshot` if purpose is unclear) → eval using setter pattern (`HTMLInputElement.prototype` value setter + input event) to batch-fill all inputs → select a non-default item for select/dropdown → click submit. Do not execute `input` field by field
|
||||
4. **Wait 1~2s**: Give time for async event handlers to fire
|
||||
5. **Stop HAR recording**: `network har stop {path}`, extract non-GET request details from HAR
|
||||
6. **Restore online and immediately leave**: `network offline off` → immediately navigate to another page to prevent retry logic from re-sending requests
|
||||
|
||||
### Information Extracted from HAR
|
||||
|
||||
For each non-GET request, record:
|
||||
- Endpoint URL + HTTP method (POST/PUT/PATCH/DELETE)
|
||||
- Request body structure (field names, types, required/optional)
|
||||
- **Input conditions at capture time**: What values were filled that triggered this request (different inputs may produce different request structures; recording input conditions helps verify generality)
|
||||
|
||||
### GraphQL Mutation Identification
|
||||
|
||||
Request body contains a `query` field starting with `mutation` → GraphQL write operation. Extract mutation name and variables structure.
|
||||
|
||||
---
|
||||
|
||||
## Operation Feasibility Assessment
|
||||
|
||||
Based on request details captured from HAR, select the strategy path:
|
||||
|
||||
**API direct connection feasible** (all of the following are satisfied):
|
||||
- All required fields in the request body have clear sources and can be parameterized
|
||||
- Serialization is transparent (request body can be reproduced directly from parameters, no opaque encoding)
|
||||
- No dynamic credentials found in HAR
|
||||
|
||||
→ Use API submission strategy, record endpoint URL + request structure.
|
||||
|
||||
> **Verification failure fallback**: Subsequent scripted verification discovers dynamic credentials or opaque encoding missed by HAR analysis → fall back to [DOM fallback submission].
|
||||
|
||||
**API incompatible** (any of the following):
|
||||
- HAR reveals frontend dynamically generated Tokens, Nonces, or HMAC signatures
|
||||
- Request body contains opaque encoding (cannot be reproduced directly from form fields)
|
||||
|
||||
→ **Fall back to DOM operation submission**, execute the "DOM Fallback Submission" steps below.
|
||||
|
||||
---
|
||||
|
||||
## DOM Fallback Submission
|
||||
|
||||
This path is taken when API is incompatible. Let the page's native JS handle credential generation; only automate form filling and submission triggering.
|
||||
|
||||
### Step 1 — Control Locating
|
||||
|
||||
**When HAR field information is available (preferred)**: Directly use HAR request body field names to locate corresponding controls (`name`/`id` usually match field names), use a single eval to batch-find, skipping full scan:
|
||||
|
||||
```javascript
|
||||
// Batch locate by field name
|
||||
const fields = ['{field_name_1}', '{field_name_2}'];
|
||||
JSON.stringify(fields.map(f => {
|
||||
const el = document.querySelector(`[name="${f}"], [id="${f}"]`);
|
||||
return el ? { field: f, tag: el.tagName, type: el.type, found: true } : { field: f, found: false };
|
||||
}))
|
||||
```
|
||||
|
||||
**When no field information is available or controls remain unknown (fallback)**: Perform a full scan of the form, returning a complete mapping of all controls (traverse upward from controls to find the nearest container for label text, do not hardcode component library class names):
|
||||
|
||||
```javascript
|
||||
// Full scan of form controls
|
||||
JSON.stringify(
|
||||
Array.from(document.querySelectorAll('input, select, textarea')).map(el => ({
|
||||
tag: el.tagName, type: el.type, name: el.name, id: el.id, placeholder: el.placeholder
|
||||
}))
|
||||
)
|
||||
```
|
||||
|
||||
### Step 2 — Assess Scriptability
|
||||
|
||||
Evaluate each field individually:
|
||||
- Value is fixed or parameterizable → **[Script]**: Fill using eval
|
||||
- Requires real-time judgment (dynamic dropdowns, captchas, complex interactions) → **[AI intervention]**: Agent performs visual operations
|
||||
|
||||
If all fields require AI intervention → produce AI Workflow strategy (see Step 3b).
|
||||
|
||||
**Multi-step forms (wizards / stepped flows)**: Page state changes after each step submission, requiring progressive advancement. Re-scan currently visible controls at each step, recording state dependencies between steps (previous step's selection affects which fields appear in the next step), so the complete operation sequence can be reproduced in the generated Skill.
|
||||
|
||||
### Step 3a — Scripted Submission Verification (when [Script] steps exist)
|
||||
|
||||
Use the safety verification protocol (HAR offline mode) to confirm the script can correctly fill and trigger submission:
|
||||
|
||||
```javascript
|
||||
// [Script] Batch fill all fields (setter pattern triggers framework reactive updates)
|
||||
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set;
|
||||
[
|
||||
['{selector_1}', '{param_value_1}'], // Element assertion: el.name === '{field_name_1}'
|
||||
['{selector_2}', '{param_value_2}'], // Element assertion: el.name === '{field_name_2}'
|
||||
].forEach(([sel, val]) => {
|
||||
const el = document.querySelector(sel);
|
||||
setter.call(el, val);
|
||||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
});
|
||||
```
|
||||
|
||||
Then trigger submission, confirm from HAR that the request was sent and the structure matches expectations.
|
||||
|
||||
> **Verification failure fallback**: HAR did not capture the expected request (framework not responding to setter, selector invalid, submission not triggered, etc.) → Upgrade failed fields to [AI intervention], re-execute Step 3a (remaining script fields) + 3b (upgraded AI fields) in mixed execution. If all fields fail → entirely transfer to Step 3b.
|
||||
|
||||
### Step 3b — AI Workflow (when all fields require AI intervention)
|
||||
|
||||
The Agent walks through the operation flow step by step like a user.
|
||||
|
||||
**Writing standards**:
|
||||
1. Each step uses browser-act subcommands in abstract form (no `browser-act` prefix — Agent adds session/flags at runtime), followed by supplementary context, ending with `→ expected result`
|
||||
2. Element references use only visual descriptions; CSS selectors and DOM characteristics are **prohibited**
|
||||
- ✗ `click button[type="submit"]`
|
||||
- ✓ `state` locate the blue "Submit" button at the bottom of the form → `click <index>`
|
||||
3. Record state checkpoints after key steps (URL, page title, visible element characteristics)
|
||||
4. Submission steps must include the complete safety verification sequence:
|
||||
`network har start` → `network offline on` → click submit → `network har stop {path}` → `network offline off`
|
||||
|
||||
Example:
|
||||
```
|
||||
1. state locate {visually described input field} → input <index> "{param_value}"
|
||||
2. screenshot → confirm input has been filled (checkpoint)
|
||||
3. state locate the "Submit" button at the bottom of the form
|
||||
4. network har start
|
||||
network offline on
|
||||
click <index>
|
||||
(wait 1~2s)
|
||||
network har stop tmp/{name}.har
|
||||
network offline off → immediately navigate to another page
|
||||
```
|
||||
|
||||
> **Verification failure**: Both scripted submission and AI Workflow fail to correctly trigger the submission request → Report current obstacles and attempted paths to the user, ask for next steps.
|
||||
|
||||
---
|
||||
|
||||
## Parameter Enumeration Collection (Required)
|
||||
|
||||
During the verification process, you **must proactively explore** every enumeration-type control on the page (dropdowns, selectors, radio groups) to determine the **data source and retrieval method** for their options. Do not record specific values — values change; recording retrieval methods is more durable.
|
||||
|
||||
Try in order of priority `API > DOM > AI`:
|
||||
|
||||
1. **Independently read current page traffic**: `network requests --filter {domain keyword}` to check if there are requests returning enumeration lists. If traffic is empty (navigated away due to HAR protocol), navigate back to the target page first
|
||||
2. **Expand/interact with the control**, observe whether async requests are triggered → if new API requests appear, record as above. If the control supports search (input field + dropdown linkage), enter a keyword to trigger the search API
|
||||
3. **API endpoint exists** (hit in step 1 or 2) → Record endpoint URL + request method + response structure (field names, value formats), verify whether independent invocation is feasible
|
||||
4. **Independent invocation not feasible** → Record as hybrid approach: what prerequisite dependencies need to be obtained from the page + how to obtain them + then how to call the API
|
||||
5. **No API endpoint** → eval to read option values from DOM (native `<select>` reads `.options`, component library controls try reading instance properties, if that fails expand and read rendered results), record selector and method
|
||||
6. **DOM also cannot reliably obtain** (multiple attempts failed or control is highly dynamic) → **[AI] approach**: Describe the visual interaction operations the Agent needs to perform
|
||||
|
||||
#### Conditional Enumeration (Cascading Linkage)
|
||||
|
||||
When cascading relationships are found between enumeration controls (selecting A causes B's options to appear/change), record the dependency chain (A→B→C) and the retrieval method for each level.
|
||||
|
||||
#### Writing to Generated Skill
|
||||
|
||||
In the generated SKILL.md's "Enum Parameters" section, provide the retrieval approach for each enumeration layered by `[API]` > `[DOM]` > `[AI]`. `[API]` and `[DOM]` must provide executable JS code; `[AI]` describes the interaction operations the Agent needs to perform (used when code cannot obtain the values). Must cover: retrieval method for each enumeration + dependency order when cascading exists. Operation steps also retain inline comments `// {param_name} enumeration retrieval: ...` to provide in-place context.
|
||||
|
||||
Enumerations that cannot be collected are marked `[collection failed]` and continue without blocking exploration.
|
||||
@@ -0,0 +1,372 @@
|
||||
# Generated Skill File Template
|
||||
|
||||
This file is read during Phase 3 execution. Create the output Skill directory according to this specification.
|
||||
|
||||
---
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
output/{skill-name}/{site-slug}-{capability-slug}/
|
||||
├── SKILL.md
|
||||
└── scripts/
|
||||
└── {capability-name}.py
|
||||
```
|
||||
|
||||
**Naming Rules**:
|
||||
- `site-slug`: Target site's primary domain, lowercase, without `www.` and TLD (e.g., `github`, `notion`, `jira`)
|
||||
- `capability-slug`: Short English description of the capability, kebab-case (e.g., `list-issues`, `create-task`, `search-users`)
|
||||
- `scripts/*.py` filenames also use kebab-case (e.g., `search-products.py`)
|
||||
- Only lowercase letters, digits, and hyphens are allowed; no underscores
|
||||
|
||||
JS code is encapsulated in Python files under `scripts/`; SKILL.md invokes them via command line. Non-JS content (Network capture steps, AI Workflow, text descriptions) remains inline in SKILL.md.
|
||||
|
||||
---
|
||||
|
||||
## SKILL.md Template
|
||||
|
||||
````markdown
|
||||
---
|
||||
name: {site-slug}-{capability-slug}
|
||||
description: "{Function statement — site name + capability + input/output overview}. Use when user mentions {site name variants}, {data type keywords}, or says {trigger phrases covering casual/formal/abbreviated expressions — as many as needed for full coverage}. Also applies to {adjacent scenarios that aren't obvious but should trigger}."
|
||||
---
|
||||
|
||||
# {site-name} — {capability-name}
|
||||
|
||||
> {one-line description: input → output}
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
{what this capability aims to achieve, one sentence}
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Target page is already open in the browser: `{full URL of the target page}`
|
||||
- {e.g., already logged in, user avatar or username is visible on the page} (when login is required)
|
||||
|
||||
<!-- Assembly guidance: Prerequisites only describe website state (page opened, logged in, etc.) and dependencies (e.g., browser extensions), not connection methods or browser types — the generation environment ≠ the runtime environment; these are decided by the caller. -->
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
<!-- Assembly guidance: Pre-execution checks only include the following fixed items; do not add custom check steps for browser connection, extension detection, etc. — those are handled by the caller based on prerequisites. -->
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
### 2. Login Verification (when prerequisites include login requirement)
|
||||
|
||||
If login status for the target site has been confirmed in the current session → skip this step.
|
||||
|
||||
Otherwise: open the target site and observe the page login status:
|
||||
- Logout/sign-out entry, user avatar, or username exists → logged in, continue execution
|
||||
- Login/register entry exists with no logout entry → not logged in, inform the user that login is needed first, assist the user in completing the login flow
|
||||
|
||||
User refuses or cannot log in → terminate execution.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page, never bypassing authentication or access controls. Its role is equivalent to copy-pasting on the user's behalf — the data is already on screen, automation merely saves time. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. `$(...)` is bash syntax; it is recommended to use the bash tool for execution.
|
||||
|
||||
Below are all atomic capabilities discovered and verified during the exploration phase, listed by command template with parameters. Simply invoke them as needed — no need to read `scripts/*.py` source code or re-verify. Only inspect scripts when execution fails for troubleshooting. Combine freely as needed during execution.
|
||||
|
||||
### API: {capability description, e.g., "get product list"}
|
||||
|
||||
`eval "$(python scripts/{capability-name}.py '{param1}' --param2 {param2})"`
|
||||
|
||||
Parameters:
|
||||
- {param1}: {description}
|
||||
- --param2: {description}, default {default-value}
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"{field-name}": "{example-value}", // {description}
|
||||
"{field-name}": 0, // {description}
|
||||
"{field-name}": null // {description}, null when no data
|
||||
}
|
||||
```
|
||||
|
||||
### API: {capability description, e.g., "submit order"}
|
||||
|
||||
`eval "$(python scripts/{capability-name}.py '{param1}' --field1 '{value1}' --field2 '{value2}')"`
|
||||
|
||||
Parameters:
|
||||
- {param1}: {description}
|
||||
- --field1: {description}
|
||||
- --field2: {description}
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"{field-name}": "{example-value}", // {description}
|
||||
"{field-name}": 0, // {description}
|
||||
"{field-name}": null // {description}, null when no data
|
||||
}
|
||||
```
|
||||
|
||||
### Network Capture: {capability description, e.g., "get search results"} (when parameters are injected via URL navigation)
|
||||
|
||||
Parameters are injected via URL; API responses are read from traffic (requests contain dynamic signatures, cannot be fetched directly):
|
||||
|
||||
1. `navigate {URL pattern, e.g., https://example.com/search?q={keyword}&page={page}}`
|
||||
2. `wait stable`
|
||||
3. `network requests --type xhr,fetch --filter {endpoint keyword}`
|
||||
4. `network request <id>`
|
||||
|
||||
Endpoint characteristic: URL contains `{characteristic path, e.g., /api/search}`
|
||||
|
||||
Error handling: When no matching target request is found, check page status (whether blocked by anti-scraping, whether login is needed, whether navigation reached the correct page), then retry once after ruling out issues.
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"{field-name}": "{example-value}", // {description}
|
||||
"{field-name}": 0, // {description}
|
||||
"{field-name}": null // {description}, null when no data
|
||||
}
|
||||
```
|
||||
|
||||
### Network Capture: {capability description, e.g., "get filtered results"} (when parameters are injected via UI operations)
|
||||
|
||||
Parameters are injected via UI operations; API responses are read from traffic (requests contain dynamic signatures, cannot be fetched directly):
|
||||
|
||||
1. {UI operation steps, e.g., fill search box, select filter criteria}
|
||||
2. Trigger request ({trigger method, e.g., click search button})
|
||||
3. `wait stable`
|
||||
4. `network requests --type xhr,fetch --filter {endpoint keyword}`
|
||||
5. `network request <id>`
|
||||
|
||||
Endpoint characteristic: URL contains `{characteristic path}`
|
||||
|
||||
Error handling: When no matching target request is found, check page status (whether blocked by anti-scraping, whether login is needed, whether navigation reached the correct page), then retry once after ruling out issues.
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"{field-name}": "{example-value}", // {description}
|
||||
"{field-name}": 0, // {description}
|
||||
"{field-name}": null // {description}, null when no data
|
||||
}
|
||||
```
|
||||
|
||||
### DOM: {data area description, e.g., "product list"} (data extraction type)
|
||||
|
||||
<!-- Assembly guidance: If the target data is asynchronously injected (browser extensions, lazy loading, etc.), add `wait --selector "{target-selector}" --state attached --timeout {milliseconds}` before the extraction command to wait for target elements to appear before extracting. -->
|
||||
|
||||
Extract: `eval "$(python scripts/{extraction-capability-name}.py)"`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"{field-name}": "{example-value}", // {description}
|
||||
"{field-name}": 0, // {description}
|
||||
"{field-name}": null // {description}, null when no data
|
||||
}
|
||||
```
|
||||
|
||||
Pagination: `eval "$(python scripts/{pagination-capability-name}.py)"`
|
||||
|
||||
[AI Intervention] {step requiring visual judgment, e.g., "confirm new data has loaded"}:
|
||||
`screenshot` confirm page change → re-run extraction script
|
||||
|
||||
### DOM: {control description, e.g., "submit form"} (operation type)
|
||||
|
||||
<!-- Assembly guidance: If form controls are asynchronously rendered (component libraries, extension injection, etc.), add `wait --selector "{control-selector}" --state attached --timeout {milliseconds}` before the fill command to ensure controls are ready. -->
|
||||
|
||||
Fill and submit: `eval "$(python scripts/{operation-capability-name}.py '{param1}' --field '{value}')"`
|
||||
|
||||
Parameters:
|
||||
- {param1}: {description}
|
||||
- --field: {description}
|
||||
|
||||
[AI Intervention] {step requiring dynamic judgment, e.g., selecting a dynamic dropdown item}:
|
||||
{judgment basis: what signal on the page determines the operation}
|
||||
→ `state` get element index then `click <index>`
|
||||
|
||||
### AI Workflow: {capability description, e.g., "browse products and extract prices"} (pure visual, used when static scripts cannot cover)
|
||||
|
||||
Each step uses browser-act subcommands (abstract form, Agent adds session/flags at runtime); element references use only visual descriptions, **no** CSS selectors or DOM characteristics; record state checkpoints after key steps:
|
||||
|
||||
1. `navigate {url}` → page loaded, title is "{expected-title}"
|
||||
2. `state` locate {visual description, e.g., "product card area with price labels in the middle of the page"} → `get text <index>`, extract `{field-name}`
|
||||
3. `scroll down` → wait for more products to load (checkpoint: new products appear)
|
||||
4. `get markdown` extract `{field-list}` from returned content
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"{field-name}": "{example-value}", // {description}
|
||||
"{field-name}": 0, // {description}
|
||||
"{field-name}": null // {description}, null when no data
|
||||
}
|
||||
```
|
||||
|
||||
### Composite: {full capability description, e.g., "get complete product data (API + DOM supplement)"}
|
||||
|
||||
<!-- Assembly guidance: Used when a single atomic component cannot provide complete data; can combine across pages (e.g., list page capture + detail page extraction + merge). Combinations that are entirely same-page JS should be merged into one Python script; when navigation or non-JS steps are involved, list steps sequentially, which may include page navigation and loops. Atomic components are still retained for individual invocation. -->
|
||||
|
||||
{When all-JS combination}:
|
||||
|
||||
`eval "$(python scripts/{composite-capability-name}.py '{param1}' --param2 {param2})"`
|
||||
|
||||
Parameters:
|
||||
- {param1}: {description}
|
||||
- --param2: {description}
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"{field-name}": "{example-value}", // {description}
|
||||
"{field-name}": 0, // {description}
|
||||
"{field-name}": null // {description}, null when no data
|
||||
}
|
||||
```
|
||||
|
||||
{When cross-page / contains non-JS steps combination}:
|
||||
|
||||
1. `navigate {page-A URL}` → `wait stable` → `eval "$(python scripts/{capability-A}.py '{param}')"`
|
||||
2. `network requests --filter {keyword}` → `network request <id>`
|
||||
3. For each `{item}` from step 1/2:
|
||||
a. `navigate {page-B URL pattern}` → `eval "$(python scripts/{capability-B}.py '{item}')"`
|
||||
4. Merge: associate by `{association-field}`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"{field-name}": "{example-value}", // {description}
|
||||
"{field-name}": 0, // {description}
|
||||
"{field-name}": null // {description}, null when no data
|
||||
}
|
||||
```
|
||||
|
||||
## Enum Parameters
|
||||
|
||||
<!-- Assembly guidance: Group by parameter; priority: API > DOM > AI. What can be obtained via code should not be left to AI. When a single endpoint can cover multiple parameters, merge into one block. -->
|
||||
|
||||
[API] {param-name} — `eval "$(python scripts/enum_{param-name}.py)"`
|
||||
|
||||
[DOM] {param-name} — `eval "$(python scripts/enum_{param-name}.py)"`
|
||||
|
||||
[AI] {param-name}: {description of what requires Agent interaction to obtain, no JS, not encapsulated}
|
||||
|
||||
Cascade dependency: {param A} → {param B} (obtain A's value first, then obtain B)
|
||||
|
||||
{param-name} [collection failed]: {failure reason}
|
||||
|
||||
## Pagination
|
||||
|
||||
<!-- Assembly guidance: Required for list-type data; delete this section for non-list types. Only keep types verified during the exploration phase; delete the rest. -->
|
||||
|
||||
**API Pagination**: `{pagination-param-name}`, type: `{page-number / cursor}`, start value: `{start-value}`. Next page value source: `{increment / response field path}`. Termination: `{termination-condition}`.
|
||||
|
||||
**URL Pagination**: URL pattern `{URL pattern}`, next page link selector: `{selector}`. Termination: `{termination-condition}`.
|
||||
|
||||
**DOM Pagination**: Click "{control description}" (`{selector}`), wait for loading then re-extract. Termination: `{termination-condition}`.
|
||||
|
||||
**AI Pagination**: Pagination driven by Agent visual operations. Each page: `screenshot` judge status → pagination operation → wait for loading → re-extract. Termination signal: `{termination-signal}`.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`{quantifiable condition expression; purely descriptive criteria are prohibited}`
|
||||
|
||||
Quantifiable dimension references:
|
||||
- Data count: `result count >= 1`
|
||||
- Field completeness: `core field non-null rate = 100%`
|
||||
- Data consistency: `matches the first N items displayed on the page`
|
||||
- Operation result: `response status 200 with no error field`
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- {e.g., rate limited to 60 requests per minute}
|
||||
- {e.g., can only query data under own account}
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script to loop through the command templates serially within a single session; do not parallelize within one browser (prone to triggering anti-scraping restrictions). Refer to rate information in "Known Limitations" above to add appropriate intervals. To increase throughput, open multiple stealth browser sessions and distribute work across them — each session has an independent fingerprint so rate limits apply per session
|
||||
- **Test before batch execution**: After writing a batch script, you must first test with 1-2 items to verify the script runs correctly; only then run the full batch. Never skip testing and execute in batch directly
|
||||
- **Reduce redundant pre-operations**: When multiple steps depend on the same prerequisite state, complete them in batch under that state to avoid repeatedly establishing the same state
|
||||
- **Error resumption**: Save results item by item during batch processing; on failure, resume from the breakpoint rather than starting over
|
||||
|
||||
## Experience Notes
|
||||
|
||||
<!-- Assembly guidance: Required universal section; output in the current language, only replace {skill-name} and {site}-{capability} in the path -->
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/{skill-name}-{site-slug}-{capability-slug}.memory.md` (working directory is determined by the Agent running the Skill, typically the project root or current working directory)
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file. Do not record what keywords were used or how many results were returned — those are task outputs, not experience.
|
||||
````
|
||||
<!-- Assembly guidance: The Experience Notes section is a universal template; output in the current language during assembly and replace path variables without modifying semantics. Strategy implementation details (API parameters, pagination methods, etc.) should be inline in their corresponding strategy reference files, not in experience notes. -->
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Python Wrapper File Template
|
||||
|
||||
Each `scripts/*.py` file follows this structure:
|
||||
|
||||
```python
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('{positional-param}') # {description}
|
||||
parser.add_argument('--{named-param}', default='{default-value}') # {description}
|
||||
args = parser.parse_args()
|
||||
|
||||
js = f"""
|
||||
(function() {{
|
||||
try {{
|
||||
// Original JS verified during exploration phase
|
||||
// Business parameters injected via f-string: {args.positional_param}, {args.named_param}
|
||||
return JSON.stringify({{ /* normal result */ }});
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{ error: true, message: e.message }});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
```
|
||||
|
||||
<!-- Assembly guidance: Python files only handle parameterized assembly of JS strings. Verified JS goes into the f-string as-is; only business parameters (keywords, page numbers, sort order, etc.) are replaced with argparse parameters. Selectors, field mappings, endpoint URLs, and other fixed values are hardcoded directly in the JS. -->
|
||||
|
||||
---
|
||||
|
||||
## Filling Specifications
|
||||
|
||||
1. **No placeholders left behind**: All `{...}` are replaced with real values; leaving blanks is not allowed
|
||||
2. **Code must be runnable**: JS strings in `scripts/*.py` must be executable directly in the browser console; Python file execution stdout must be valid JS
|
||||
3. **Use placeholders for runtime variables**: Keywords, business parameters, pagination offsets, etc. use `{param-name}` and are not hardcoded with specific values. If demonstrating typical usage, provide a separate "Usage Example" block outside the code block with a note, separate from the strategy code itself
|
||||
4. **Description must be entirely in English** (every word — function statement, trigger phrases, scenarios — even when target users speak Chinese / Japanese / other languages; never embed non-English keywords or phrases). Three parts: function statement (site + capability + I/O), trigger scenarios (keyword-rich English phrases covering site name variants, data keywords; think from the user's perspective: what English words would a user use to mention this capability? Casual, formal, abbreviated — include all real-world phrasings), and scope expansion (adjacent non-obvious triggers). Site name goes first for matching priority; be concise but don't sacrifice keyword coverage — aim for under 1024 characters (the platform limit); no markdown formatting. Write in third person, err toward being "pushy" (Claude tends to under-trigger)
|
||||
5. **Limitations must be real**: Only document limitations actually encountered during exploration; do not speculate
|
||||
6. **Add/remove components as needed**: Each API / Network Capture / DOM entry under "Capability Components" is added or removed based on actual exploration results; no fixed quantity is required
|
||||
7. **Enum priority order**: For the same parameter, list `[API]` first then `[DOM]`; if only one exists, list only that one
|
||||
8. **Mark collection failures**: Enums that cannot be collected are marked with `{param-name} [collection failed]: {reason}`; do not leave blank
|
||||
9. **Delete sections as needed**: Delete "Enum Parameters" when there are no enums; delete "Pagination" when not a list type
|
||||
10. **Enums do not duplicate capability components**: If an enum's retrieval method already exists as a capability component, the Enum Parameters section directly references that component (e.g., "retrieval method: see API component above: {component-name}"); do not duplicate code
|
||||
11. **Do not include task-specific instance data or origin references**: A Skill is a reusable template; it must not contain specific inputs from this exploration task (search keywords, specific URL lists, usernames, etc.) or reference materials provided by the user (competitor links, third-party tool descriptions, requirement background). When the request was to reproduce capabilities of an existing scraper / SaaS / data extraction platform (e.g. Apify, Octoparse, Bright Data, ScrapingBee, Phantombuster, Diffbot, etc.), the generated Skill must NOT mention the source platform anywhere — no "reproduces X actor", no "equivalent to Y scraper", no source platform name in description / objective / capability titles / comments / examples. The Skill describes the target site capability as a standalone reusable template, as if the source platform was never referenced.
|
||||
12. **Strip assembly guidance**: All HTML comments (`<!-- ... -->`) in the template are generation guidance and must not appear in the generated Skill output
|
||||
13. **One JS snippet per .py file**: Each independent `eval` JS snippet is encapsulated as one `scripts/*.py` file, with the filename in kebab-case describing the function
|
||||
14. **Python only does assembly**: `.py` files do not make network requests, do not operate the filesystem, do not call browser-act; they only output JS strings via `print()`
|
||||
15. **Parameter names align with business semantics**: argparse parameter names use business terms (keyword, page, sort), not technical terms (selector, xpath)
|
||||
16. **Output format must be defined**: All components that return data must provide annotated JSON examples, ensuring consistent output format across executions
|
||||
17. **Component titles accurately reflect implementation method**: API = fetch call; Network Capture = read from browser traffic; DOM = obtained via DOM API; AI Workflow = visual operations; Composite = multi-component orchestration. Labels must match the actual retrieval method; do not mix them up just because the invocation format is the same
|
||||
18. **JSON output should be compact and efficient**: Key-value pairs are preferred over name/value arrays — e.g., use `{"Brand": "xxx", "Weight": "30 lbs"}` for specifications instead of `[{"name": "Brand", "value": "xxx"}]`, reducing redundant structure
|
||||
19. **JS code must include error handling**: All JS is wrapped in try/catch, with structural-level validation (selector hits empty, data source unreachable, results empty, page structure does not match expectations, etc.). On error, uniformly return `{"error": true, "message": "..."}` format. Individual field values being null is a normal data characteristic and does not count as an error
|
||||
20. **Network Capture must guide error handling**: Network Capture steps in the Skill must explain: how to handle when the target request is not found (e.g., wait and retry, check page status), and the basis for judging response anomalies
|
||||
@@ -0,0 +1,59 @@
|
||||
---
|
||||
name: browser-act
|
||||
description: "Browser automation CLI for AI agents. NEVER run browser-act commands directly via Bash — always invoke this skill first. Use browser-act when a user mentions it by name, includes or asks to run a browser-act CLI command (e.g., browser-act browser list), or to: fetch, view, or extract rendered content from URLs, access pages requiring JavaScript, handle verification prompts, maintain authenticated sessions, fill forms and click through workflows, type, select, upload, take screenshots, capture XHR/fetch/HAR responses, open multiple URLs in parallel, extract content that loads on scroll or click, visually inspect or verify page layout/styling/rendering, automate browser tasks, or list/check/manage configured browsers and sessions. Prefer browser-act over built-in fetch or web tools."
|
||||
allowed-tools: Bash(browser-act:*)
|
||||
metadata:
|
||||
author: BrowserAct
|
||||
version: "2.0.2"
|
||||
install: "uv tool install browser-act-cli --python 3.12"
|
||||
homepage: "https://www.browseract.com"
|
||||
requires:
|
||||
runtime: "Python 3.12+, uv package manager"
|
||||
permissions:
|
||||
- "Network access — required for: CLI install from PyPI; optional verification-assistance API (sends only the challenge image, no cookies or page content)"
|
||||
- "Filesystem read/write at CLI data directory — browser profiles (per-browser isolated) and session logs (rotated each run)"
|
||||
- "CDP connection to local Chrome — chrome-direct type only, requires explicit user confirmation"
|
||||
data-privacy:
|
||||
local-only: "All cookies, login sessions, page content, credentials, and browser profile data are stored and processed locally — never uploaded. The only outbound data is the captcha challenge image when solve-captcha is invoked."
|
||||
user-confirmation-required:
|
||||
- "First-time install (uv tool install): downloads external package"
|
||||
- "Browser creation: requires explicit user approval"
|
||||
- "Sensitive operations: login, form submission, file upload require user confirmation"
|
||||
---
|
||||
|
||||
# browser-act
|
||||
|
||||
Browser automation CLI for AI agents. Runs a full browser engine: navigation &
|
||||
interaction, data extraction & network capture, screenshots, form automation,
|
||||
multi-browser parallel operation, user-configured proxy support, and
|
||||
human-agent collaboration.
|
||||
|
||||
### Features
|
||||
|
||||
- Lightweight extraction — fast JS-rendered content fetch without opening a browser session, advanced WebFetch/curl replacement
|
||||
- Session management — multi-browser isolation, multi-account parallel operation
|
||||
- Verification assistance — when automation encounters interactive challenges, assists completion with user authorization
|
||||
- Complex interaction — DOM content extraction, screenshots, form filling, file upload
|
||||
- Human-agent collaboration — headed mode + remote assist for manual steps
|
||||
- Safety controls — Confirmation Gate protocol requires explicit user approval before browser creation, deletion, and sensitive operations
|
||||
- Universal compatibility — works with Cursor, Claude Code, Codex, Windsurf, etc.
|
||||
|
||||
Install: `uv tool install browser-act-cli --python 3.12`
|
||||
|
||||
## Start here
|
||||
|
||||
Before running any `browser-act` command, load the usage guide from the CLI:
|
||||
|
||||
```bash
|
||||
browser-act get-skills core --skill-version 2.0.2 # start here — workflows, common patterns, troubleshooting
|
||||
```
|
||||
|
||||
**Do NOT skip this step regardless of how simple the command seems.**
|
||||
|
||||
**Do NOT truncate the output** — it contains operational directives and
|
||||
environment state that are critical for correct operation. Truncating will
|
||||
cause you to miss browser selection rules and safety constraints.
|
||||
|
||||
`get-skills core` provides environment status, available browsers, operational
|
||||
directives, and the complete interaction workflow — none of which are available
|
||||
through `--help`.
|
||||
@@ -0,0 +1,26 @@
|
||||
# BrowserAct Documentation
|
||||
|
||||
Browser automation CLI built for AI agents. Get past anti-bot walls, hand off to humans across platforms when stuck, run parallel tasks without cross-contamination, and isolate multiple accounts in independent browsers.
|
||||
|
||||
## Getting Started
|
||||
|
||||
- [Installation](installation.md) — Agent integration or manual install
|
||||
- [Quick Start](quick-start.md) — 60 seconds to first result
|
||||
|
||||
## Features
|
||||
|
||||
- [Anti-Blocking](anti-blocking.md) — Three-layer system: environment-layer emulation / execution-layer auto-resolution / human-layer remote handoff
|
||||
- [Better Headless](headless.md) — Default headless without disturbing, stealth headless undetected, no need to switch to headed when humans are needed
|
||||
- [Remote Assist](remote-assist.md) — One command to hand off to a human, any device takes over via the link, agent resumes seamlessly when done
|
||||
- [Browser Modes](browser-modes.md) — chrome (reuse local login), stealth privacy (frictionless batch scraping), stealth fixed identity (multi-account uncorrelated)
|
||||
- [Concurrency & Isolation](concurrency.md) — Independent identity across browsers, multi-session workspaces within a browser, zero residue in privacy mode
|
||||
- [Agent Design](agent-design.md) — Compact text output, indexed interaction, desc semantic memory, 50+ complete automation commands, secure by default
|
||||
|
||||
## Your Personal Scraping Engineer
|
||||
|
||||
- [Skill Forge](skill-forge.md) — Explore once, generate a deploy-ready Skill, run the same stable path repeatedly — let AI write your scrapers
|
||||
|
||||
## Integration & Reference
|
||||
|
||||
- [Skills](skills.md) — Entry Skill + get-skills mechanism
|
||||
- [Commands](commands.md) — Complete command reference
|
||||
@@ -0,0 +1,169 @@
|
||||
# Agent Design
|
||||
|
||||
Not for humans writing scripts — for models picking actions. Every interface decision starts from "how does an LLM use this?"
|
||||
|
||||
## Design Philosophy
|
||||
|
||||
### Compact Text Output
|
||||
|
||||
Page state, screenshots, and command results are returned in token-friendly format — not human-formatted JSON.
|
||||
|
||||
Compact indexed text is several times more token-efficient than full DOM HTML or formatted JSON, letting the agent run more tasks within a limited context.
|
||||
|
||||
```
|
||||
url=https://example.com/login
|
||||
title=Login
|
||||
|
||||
*[1]<div id=login-form />
|
||||
*[2]<input type=email placeholder=Email />
|
||||
*[3]<input type=password placeholder=Password />
|
||||
*[4]<button id=submit />
|
||||
Sign In
|
||||
```
|
||||
|
||||
### Indexed Interaction
|
||||
|
||||
`state` returns an indexed list of interactive elements (the `N` in `[N]` in the output is the element index). The agent reasons directly with indices:
|
||||
|
||||
```bash
|
||||
browser-act --session s1 click 3 # Click element at index 3
|
||||
browser-act --session s1 input 2 "hello" # Type into element at index 2
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
- No XPath, CSS selectors, or DOM parsing
|
||||
- The agent sees exactly what it can operate
|
||||
- Indices update with the page — re-run `state` for fresh ones
|
||||
|
||||
### desc Semantic Memory
|
||||
|
||||
Every browser carries a `desc` field — a natural-language description of its purpose. In a new conversation, the agent matches tasks to browsers by meaning, with no hardcoded IDs to drag across sessions.
|
||||
|
||||
```
|
||||
"Taobao shopping account, logged in as user123. Used for price monitoring."
|
||||
```
|
||||
|
||||
The agent uses `desc` to:
|
||||
- Match browsers to tasks by meaning ("check Taobao prices" → Taobao browser)
|
||||
- Avoid recreating browsers for known workflows
|
||||
- Accumulate purpose knowledge for each browser over time
|
||||
|
||||
Use append semantics to preserve history:
|
||||
|
||||
```bash
|
||||
browser-act browser update <id> --desc-append "Also used for order tracking"
|
||||
browser-act browser update <id> --desc "Fully overwrite the description"
|
||||
```
|
||||
|
||||
#### Browser Selection Priority
|
||||
|
||||
When the agent has multiple browsers, selection follows this priority:
|
||||
|
||||
| Priority | Condition | Behavior |
|
||||
|----------|-----------|----------|
|
||||
| 1 | `desc` clearly matches the task | Use directly, no need to ask |
|
||||
| 2 | Only one browser exists | Use directly |
|
||||
| 3 | Multiple browsers, no clear match | List candidates, let the user pick |
|
||||
|
||||
This logic is enforced at the Skill layer, not the CLI. After the user picks, the agent should append the new finding to `desc` so a similar task next time matches directly.
|
||||
|
||||
### Secure by Default
|
||||
|
||||
Letting an AI agent control a browser is powerful and potentially dangerous. The security mechanism doesn't try to block all misuse purely through technical means — instead, **confirmation gates** keep humans informed and in control.
|
||||
|
||||
The real risks of multi-agent automation come from three directions, each with a corresponding design fallback:
|
||||
|
||||
#### Concurrency Safety
|
||||
|
||||
Session ownership + explicit-naming model. Multiple agents sharing one browser don't conflict, and failures are scoped to the session they originate from. See [Concurrency & Isolation](concurrency.md).
|
||||
|
||||
#### Operational Safety: Confirmation Gating
|
||||
|
||||
Sensitive operations require explicit user approval before execution. This is enforced through the Skill — not a CLI hard-block, but a conversation-level behavior protocol.
|
||||
|
||||
> **Note:** The actual effect of confirmation gating depends on the agent architecture and underlying model capabilities you use. Different platforms and models follow Skill instructions to different degrees. We continually optimize the Skill's guidance strategy for cross-platform compatibility.
|
||||
|
||||
Operations requiring confirmation:
|
||||
|
||||
| Operation | Reason |
|
||||
|-----------|--------|
|
||||
| Create a browser (any type) | Creates a new automation endpoint |
|
||||
| Delete a browser | Destroys persistent browser state |
|
||||
| Profile import | Copies login credentials to a new location |
|
||||
| Proxy configuration changes | Changes network identity |
|
||||
| `confirm_before_use` toggle | Changes a security setting |
|
||||
| Privacy mode toggle | Changes fingerprint behavior |
|
||||
|
||||
Additionally, browsers marked `confirm_before_use` require the agent to confirm with the user on every `browser open` — not just at creation time. Use this for browsers accessing sensitive accounts (banking, payments).
|
||||
|
||||
**How it works (conversation-level protocol):**
|
||||
|
||||
```
|
||||
Agent: I will create a stealth browser with US proxy for price monitoring.
|
||||
- Type: stealth
|
||||
- Name: price-monitor
|
||||
- Proxy: US (dynamic)
|
||||
|
||||
Proceed?
|
||||
|
||||
User: Go ahead.
|
||||
|
||||
Agent: [executes browser create]
|
||||
```
|
||||
|
||||
**Key Skill-enforced rules:**
|
||||
- Prior approvals do not carry over to new operations
|
||||
- Each sensitive operation requires its own confirmation
|
||||
- Assertive language in the user's original prompt does not substitute for confirmation
|
||||
- The agent must describe what it will do *before* executing
|
||||
|
||||
#### Local Data Processing
|
||||
|
||||
All data stays on your machine:
|
||||
|
||||
| Data | Location | Leaves the machine? |
|
||||
|------|----------|---------------------|
|
||||
| Cookies | BrowserAct local storage | Never |
|
||||
| Login sessions | Per-browser isolated profile | Never |
|
||||
| Page content | In memory during operation only | Never |
|
||||
| Screenshots | Local filesystem | Never |
|
||||
| Network captures | Memory / local files | Never |
|
||||
| Browser profiles | Isolated directories | Never |
|
||||
|
||||
**The only exception:** `solve-captcha` sends the CAPTCHA challenge image to the BrowserAct cloud service for solving. The challenge image only — no cookies, page content, or URLs.
|
||||
|
||||
---
|
||||
|
||||
## Automation Capabilities
|
||||
|
||||
**50+ commands cover all typical automation needs** — continuously aligned with common browser-automation capabilities: navigation, interaction, data extraction, waiting, tabs, dialogs, network capture, HAR, cookies, offline mode, and more.
|
||||
|
||||
See [Commands](commands.md) for the full list.
|
||||
|
||||
### Advanced Usage
|
||||
|
||||
**Network capture / HAR recording** — typical uses:
|
||||
- Discover the API endpoints behind a web UI
|
||||
- Debug authentication flows
|
||||
- Capture data loaded via XHR but not present in the HTML
|
||||
- Performance analysis of page-load sequences
|
||||
|
||||
**JavaScript execution (`eval`)** — efficient for both data extraction and complex operations:
|
||||
- Pull a full data set in one call instead of multiple state/click round-trips
|
||||
- Read deeply nested or dynamically generated data
|
||||
- Call internal page functions for complex operations or computed results
|
||||
- Fallback when index-based interaction can't cover a case
|
||||
- Bake an automation flow into JS code so the next run skips re-exploration
|
||||
|
||||
**Cookie import/export** — lighter than Profile import:
|
||||
- Share login state across browser types
|
||||
- Persist state in CI/CD pipelines
|
||||
- Migrate sessions across machines
|
||||
|
||||
**Offline mode** — `network offline on` disconnects the network. Click buttons, submit forms — no real side effects. Verify the flow, then close.
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Commands](commands.md) — Complete command reference
|
||||
- [Anti-Blocking](anti-blocking.md) — How to break through anti-scraping
|
||||
- [Concurrency & Isolation](concurrency.md) — Safety guarantees for parallel agents
|
||||
@@ -0,0 +1,173 @@
|
||||
# Anti-Blocking
|
||||
|
||||
Reach places standard tools can't. **Three progressive layers, escalating only as needed.** The vast majority of blocks never reach the agent — they're absorbed at the environment layer. The few that trigger get auto-resolved at the execution layer. Only extreme cases escalate to a human.
|
||||
|
||||
## The Three-Layer System
|
||||
|
||||
```
|
||||
Environment layer (look like a real user)
|
||||
↓ triggered?
|
||||
Execution layer (auto-solve verification)
|
||||
↓ can't be solved?
|
||||
Human layer (let a human jump in — see "Remote Assist")
|
||||
```
|
||||
|
||||
| Layer | What it solves | Key capabilities |
|
||||
|-------|----------------|------------------|
|
||||
| **Environment** | Prevent verification from triggering at all | Fingerprint spoofing, TLS rotation, residential proxies, privacy mode |
|
||||
| **Execution** | Auto-resolve when triggered | `solve-captcha` auto CAPTCHA, `stealth-extract` one-command extraction |
|
||||
| **Human** | Cases automation can't handle | `remote-assist` remote handoff (see [Better Headless](headless.md)) |
|
||||
|
||||
## Environment Layer: Look Like a Real User
|
||||
|
||||
stealth browsers ship with a complete anti-detection stack, ensuring the site sees no difference from a real browser.
|
||||
|
||||
| Technique | Purpose |
|
||||
|-----------|---------|
|
||||
| **Fingerprint spoofing** | Canvas / WebGL / fonts / plugins — consistently faked, all components tell the same story |
|
||||
| **Navigator patching** | `webdriver` / `chrome.runtime` / plugin array all normalized |
|
||||
| **TLS fingerprint rotation** | Matches real-browser TLS signatures |
|
||||
| **Headless concealment** | Run headless to save resources AND pass detection — both at once |
|
||||
| **Proxy system** | Dynamic proxy (auto-rotated regional IPs) / Static proxy (managed fixed IP) / Custom proxy (BYO) |
|
||||
| **Privacy mode** | Fresh fingerprint + empty profile per session, no persistence |
|
||||
|
||||
### Proxy System
|
||||
|
||||
Only stealth browsers support proxies. The three modes are mutually exclusive:
|
||||
|
||||
**Dynamic proxy (managed)**
|
||||
|
||||
```bash
|
||||
browser-act browser create --type stealth --name s1 --desc "..." --dynamic-proxy US
|
||||
```
|
||||
|
||||
- IP auto-rotates on every browser restart
|
||||
- List available regions: `browser-act browser regions`
|
||||
|
||||
**Static proxy (managed fixed IP)**
|
||||
|
||||
```bash
|
||||
browser-act browser create --type stealth --name s1 --desc "..." --static-proxy <proxy_id>
|
||||
```
|
||||
|
||||
- Fixed IP, stable across sessions
|
||||
- Best for account warm-up, login persistence, API allowlists, and any scenario that requires being bound to the same IP long-term
|
||||
- List purchased proxies: `browser-act proxy list`
|
||||
- Purchase a new proxy: `browser-act proxy buy-request` (returns a purchase URL)
|
||||
|
||||
**Custom proxy (BYO)**
|
||||
|
||||
```bash
|
||||
browser-act browser create --type stealth --name s1 --desc "..." --custom-proxy socks5://user:pass@host:port
|
||||
```
|
||||
|
||||
- Supports HTTP / SOCKS5
|
||||
- Your proxy, your IPs
|
||||
|
||||
### Privacy Mode
|
||||
|
||||
When enabled, each session uses a fresh fingerprint and empty profile, with no data persisted:
|
||||
|
||||
```bash
|
||||
browser-act browser create --type stealth --name "ephemeral" --desc "One-off task" --private true
|
||||
```
|
||||
|
||||
Or toggle on an existing browser:
|
||||
|
||||
```bash
|
||||
browser-act browser update <stealth-id> --private true
|
||||
```
|
||||
|
||||
Use cases: multi-account isolation, preventing fingerprint accumulation, one-off operations. Trade-off: cannot retain login state.
|
||||
|
||||
## Execution Layer: Auto-Resolve Verification
|
||||
|
||||
### stealth-extract (WebFetch Replacement)
|
||||
|
||||
Read-only content extraction with zero session management:
|
||||
|
||||
```bash
|
||||
browser-act stealth-extract https://protected-site.com
|
||||
```
|
||||
|
||||
**What it does:**
|
||||
1. Launches an anti-detection browser with full fingerprint spoofing
|
||||
2. Navigates to the target URL and waits for JS to render
|
||||
3. Returns the page in markdown (or HTML)
|
||||
4. Closes the browser — no cleanup required
|
||||
|
||||
**No session, no browser management, no state.** URL in, content out.
|
||||
|
||||
```bash
|
||||
# HTML output
|
||||
browser-act stealth-extract https://example.com --content-type html
|
||||
|
||||
# Use a managed proxy (region-specific IP)
|
||||
browser-act stealth-extract https://example.com --dynamic-proxy US
|
||||
|
||||
# Use a managed static proxy (fixed IP)
|
||||
browser-act stealth-extract https://example.com --static-proxy <proxy_id>
|
||||
|
||||
# Use a custom proxy
|
||||
browser-act stealth-extract https://example.com --custom-proxy socks5://host:port
|
||||
|
||||
# Custom timeout (default 30s)
|
||||
browser-act stealth-extract https://example.com --timeout 60
|
||||
|
||||
# Save to file
|
||||
browser-act stealth-extract https://example.com --output ./result.md
|
||||
```
|
||||
|
||||
**Rule:** Only need to *read*? Use `stealth-extract`. Need to *interact*? Use a stealth browser.
|
||||
|
||||
### solve-captcha Auto CAPTCHA
|
||||
|
||||
Automatically identifies and solves common CAPTCHA types (image selection, text recognition, etc.):
|
||||
|
||||
```bash
|
||||
browser-act --session s1 solve-captcha
|
||||
```
|
||||
|
||||
Returns `solved=True` when it goes through automatically.
|
||||
|
||||
#### Escalation when auto-solving fails
|
||||
|
||||
```bash
|
||||
# Try automatic first
|
||||
browser-act --session s1 solve-captcha
|
||||
|
||||
# If it fails, show it to the user locally
|
||||
browser-act --session s1 browser open <id> <url> --headed
|
||||
|
||||
# Or escalate to remote handoff
|
||||
browser-act --session s1 remote-assist --objective "Please solve the CAPTCHA"
|
||||
```
|
||||
|
||||
## Human Layer: Hand It to a Person
|
||||
|
||||
When environment + execution still can't solve it, escalate to a human — `remote-assist` generates a live URL that the user opens on any device to take over. See [Remote Assist](remote-assist.md).
|
||||
|
||||
## Escalation Strategy
|
||||
|
||||
```
|
||||
1. Default chrome / chrome-direct + no proxy → fits most everyday sites
|
||||
2. Anti-scraping triggered → upgrade to stealth + configure a proxy
|
||||
3. CAPTCHA still appearing → solve-captcha auto-resolves
|
||||
4. Auto-resolution fails → remote-assist hands off to a human
|
||||
```
|
||||
|
||||
## Comparison with Baseline Tools
|
||||
|
||||
| | curl / WebFetch | Standard Puppeteer / Playwright | BrowserAct |
|
||||
|---|---|---|---|
|
||||
| Handles JS rendering | ✗ | ✓ | ✓ |
|
||||
| Anti-scraping bypass | ✗ | ✗ | ✓ (environment layer) |
|
||||
| Auto CAPTCHA | ✗ | ✗ | ✓ (execution layer) |
|
||||
| Human handoff when stuck | ✗ | ✗ | ✓ (human layer) |
|
||||
| Headless without detection | N/A | ✗ (headless easily detected) | ✓ (stealth headless) |
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Better Headless](headless.md) — Default headless + remote handoff
|
||||
- [Browser Modes](browser-modes.md) — Pick the right browser
|
||||
- [Skill Forge](skill-forge.md) — Bake anti-scraping experience into reusable Skills
|
||||
@@ -0,0 +1,209 @@
|
||||
# Browser Modes
|
||||
|
||||
Three browser modes by use case, all running on your machine:
|
||||
|
||||
| Mode | Scenario | Key trait |
|
||||
|------|----------|-----------|
|
||||
| **chrome** | Reuse local Chrome login state | Two sub-modes: Profile import / CDP attach |
|
||||
| **stealth · privacy mode** | Frictionless batch scraping without login | Fresh fingerprint per session + proxy rotation, zero residue |
|
||||
| **stealth · fixed identity** | Logged-in accounts · multi-browser parallel | Stable fingerprint + stable IP, stable account identity, not flagged as bots |
|
||||
|
||||
## chrome: Reuse Local Chrome Login State
|
||||
|
||||
Best for already-logged-in sites (Gmail, GitHub, Jira, etc.) when you don't want to log in again. Two sub-modes are available.
|
||||
|
||||
### Sub-mode 1: Import Local Profile
|
||||
|
||||
Extract a Profile (cookies, localStorage, IndexedDB) from local Chrome into a standalone Chromium instance:
|
||||
|
||||
```bash
|
||||
# List importable profiles
|
||||
browser-act browser list-profiles
|
||||
|
||||
# Create a chrome browser with the chosen profile
|
||||
browser-act browser create --type chrome --name "work" \
|
||||
--desc "Work Chrome: logged into GitHub, Jira, Gmail" \
|
||||
--source-profile <profile-id>
|
||||
```
|
||||
|
||||
**Properties:**
|
||||
- A standalone Chromium instance, isolated from your local Chrome
|
||||
- Import is a one-time snapshot — later changes in local Chrome do not sync
|
||||
- Quota: up to 20 browsers
|
||||
- Suitable for long-running automation tasks
|
||||
|
||||
#### You can also import after creation
|
||||
|
||||
If the browser already exists, import separately:
|
||||
|
||||
```bash
|
||||
browser-act browser import-profile <browser_id> <profile_id>
|
||||
|
||||
# If Chrome needs to be restarted to enable CDP
|
||||
browser-act browser import-profile <browser_id> <profile_id> --allow-restart-chrome
|
||||
```
|
||||
|
||||
#### What is imported
|
||||
|
||||
| Included | Excluded |
|
||||
|----------|----------|
|
||||
| Cookies | Browsing history |
|
||||
| localStorage | Bookmarks |
|
||||
| IndexedDB | Extensions |
|
||||
| Session storage | Cache |
|
||||
| | Saved passwords |
|
||||
|
||||
#### Two import modes
|
||||
|
||||
| Mode | Path | Trait |
|
||||
|------|------|-------|
|
||||
| **Local mode** | chrome → chrome | Direct file copy. Fastest, most complete |
|
||||
| **CDP mode** | any → stealth, cross-type | Network extraction via DevTools Protocol |
|
||||
|
||||
#### Prerequisites
|
||||
|
||||
- Must call `browser list-profiles` first to discover Profile IDs
|
||||
- The target browser must already exist (created via `browser create`)
|
||||
- The source browser (Chrome) may need to be closed for local import
|
||||
|
||||
#### Risk Notes
|
||||
|
||||
Profile import has inherent risks the agent must communicate to users:
|
||||
|
||||
1. **The source browser may be closed during import.**
|
||||
2. **An IP change at the new location may trigger re-verification on some sites.**
|
||||
3. **Environment differences (fingerprint, location) may trigger re-login.**
|
||||
4. **Import is a snapshot — later changes at the source do not propagate.**
|
||||
|
||||
### Sub-mode 2: CDP Direct Attach
|
||||
|
||||
Directly drive your running local Chrome — extensions, certificates, and SSO are all in place:
|
||||
|
||||
```bash
|
||||
browser-act browser create --type chrome-direct --name "live" --desc "Direct attach to local Chrome"
|
||||
browser-act --session work browser open <browser-id> https://internal.corp.com
|
||||
```
|
||||
|
||||
**Properties:**
|
||||
- Zero configuration — no Profile import required
|
||||
- Full inheritance of local Chrome's extensions, bookmarks, certificates, and SSO cookies
|
||||
- Quota: 1 chrome-direct browser globally
|
||||
- While running, your Chrome is being automated (you can't use it manually)
|
||||
- Headed mode is not supported (since it's already your browser)
|
||||
|
||||
**Best for:** Enterprise SSO, sites that depend on specific extensions or certificates, quick operations that don't require isolation.
|
||||
|
||||
### Comparing the Two Sub-modes
|
||||
|
||||
| | Profile import | CDP attach |
|
||||
|---|---|---|
|
||||
| Setup | Choose a profile to import | Zero config |
|
||||
| Isolation | Standalone process | Your actual Chrome |
|
||||
| Extensions / certs | Excluded from import | Fully inherited |
|
||||
| Quota | 20 | 1 (global) |
|
||||
| User's Chrome occupied? | No | Yes |
|
||||
| Long-running tasks | ✓ | ✗ (occupies your Chrome) |
|
||||
|
||||
## stealth · Privacy Mode: Login-Free Batch Scraping
|
||||
|
||||
Fresh fingerprint per session + auto-rotating proxy IPs, zero residue. Ideal for monitoring competitor sites at scale — prices, SKUs, new arrivals — with no traces left behind.
|
||||
|
||||
```bash
|
||||
# Create a stealth browser with privacy mode + dynamic proxy
|
||||
browser-act browser create --type stealth --name "monitor" \
|
||||
--desc "Competitor price monitoring" \
|
||||
--dynamic-proxy US \
|
||||
--private true
|
||||
```
|
||||
|
||||
**Properties:**
|
||||
- Each `browser open` session gets a fresh fingerprint and an empty profile, nothing persisted
|
||||
- Dynamic proxy auto-rotates IPs by region
|
||||
- Passes anti-detection in headless mode with full spoofing intact
|
||||
- Best for one-off tasks, high-anonymity needs, and avoiding fingerprint accumulation
|
||||
|
||||
**Trade-off:** Login state is not retained; requires an API Key (managed service).
|
||||
|
||||
## stealth · Fixed Identity: Logged-In Multi-Browser
|
||||
|
||||
Each browser keeps a stable fingerprint + stable IP, so the account looks like a real user. Scale to multiple independent browsers — accounts cannot be correlated across them.
|
||||
|
||||
Requires fixed IPs (dynamic proxies rotate). Recommended: use managed static proxies:
|
||||
|
||||
```bash
|
||||
# List purchased static proxies
|
||||
browser-act proxy list
|
||||
|
||||
# Each store gets its own stealth browser with a dedicated static proxy
|
||||
browser-act browser create --type stealth --name "shop-1" \
|
||||
--desc "Taobao store 1: women's clothing" \
|
||||
--static-proxy <proxy_id_1>
|
||||
|
||||
browser-act browser create --type stealth --name "shop-2" \
|
||||
--desc "Taobao store 2: electronics" \
|
||||
--static-proxy <proxy_id_2>
|
||||
```
|
||||
|
||||
Also supports `--custom-proxy socks5://host:port` if you bring your own fixed proxy.
|
||||
|
||||
**Properties:**
|
||||
- Each browser has independent fingerprint, fixed proxy, and independent cookies
|
||||
- Same browser keeps the same IP across uses — sites treat it as a stable real user
|
||||
- Sites cannot correlate across browsers
|
||||
- Login state persists; subsequent operations skip the login flow
|
||||
- Best for multi-store management, multi-account operations, and multi-account competitive monitoring
|
||||
|
||||
**Trade-off:** Requires an API Key (managed service). Managed static proxies require purchase; custom proxies are bring-your-own.
|
||||
|
||||
## Picking a Mode by Task
|
||||
|
||||
| Task | Pick |
|
||||
|------|------|
|
||||
| Automate a site you're already logged into in Chrome | **chrome with Profile import** |
|
||||
| Need local extensions, certificates, or SSO | **chrome-direct (CDP)** |
|
||||
| Scrape public content protected by anti-scraping | **stealth privacy mode** + proxy |
|
||||
| Run multiple independent accounts long-term | **stealth fixed identity** (one browser per account) |
|
||||
| Just need to read a page once | **stealth-extract** (no browser to create) |
|
||||
|
||||
## Real-World Switching Paths
|
||||
|
||||
**Scenario 1: From chrome to stealth**
|
||||
|
||||
> You automate an e-commerce site with chrome. After a few days, captchas start appearing. Switch to stealth privacy mode — a "clean identity" continues, avoiding correlation.
|
||||
|
||||
**Scenario 2: From stealth to chrome-direct**
|
||||
|
||||
> You log into an enterprise system with stealth, but it depends on a specific browser extension. Switch to chrome-direct to attach to your local Chrome (with the extension already installed).
|
||||
|
||||
**Scenario 3: From chrome-direct to chrome**
|
||||
|
||||
> You ran a task with chrome-direct successfully. But chrome-direct has only one quota and you don't want to occupy your local Chrome each time. Import the login state into a chrome browser and use chrome from then on.
|
||||
|
||||
## Quotas
|
||||
|
||||
| Type | Max | Notes |
|
||||
|------|-----|-------|
|
||||
| `chrome` (incl. chrome-direct) | 20 + 1 | chrome: 20 standalone processes; chrome-direct: 1 global |
|
||||
| `stealth` | Per account | Allocated based on the account |
|
||||
|
||||
## Unified Data Model
|
||||
|
||||
Regardless of mode, every browser shares the same structure:
|
||||
|
||||
| Field | Description |
|
||||
|-------|-------------|
|
||||
| `id` | Unique identifier, auto-generated |
|
||||
| `name` | Human-readable name |
|
||||
| `type` | `chrome` / `chrome-direct` / `stealth` |
|
||||
| `desc` | Natural-language purpose description (see [Agent Design](agent-design.md#desc-semantic-memory)) |
|
||||
| `dynamic_proxy` | Managed proxy region code (stealth only) |
|
||||
| `static_proxy` | Managed static proxy ID (stealth only) |
|
||||
| `custom_proxy` | Custom proxy URL (stealth only) |
|
||||
| `private` | Privacy mode toggle, default false (stealth only) |
|
||||
| `confirm_before_use` | Whether to ask the user before each use |
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Anti-Blocking](anti-blocking.md) — Anti-scraping deep dive for stealth browsers
|
||||
- [Concurrency & Isolation](concurrency.md) — Multi-browser parallel patterns
|
||||
- [Agent Design](agent-design.md) — desc semantic memory and browser selection logic
|
||||
@@ -0,0 +1,190 @@
|
||||
# Commands
|
||||
|
||||
Complete BrowserAct CLI command index. For detailed usage and context, see the relevant feature chapter.
|
||||
|
||||
## Global Options
|
||||
|
||||
| Option | Description |
|
||||
|--------|-------------|
|
||||
| `--session <name>` | Session name (required for browser interaction commands) |
|
||||
| `--format text\|json` | Output format |
|
||||
| `--no-auto-dialog` | Disable automatic dialog handling |
|
||||
| `--version` | Show CLI version |
|
||||
|
||||
## Common Tasks
|
||||
|
||||
| I want to... | Use these commands | See |
|
||||
|--------------|--------------------|-----|
|
||||
| Extract content from a protected site | `stealth-extract` | [Anti-Blocking](anti-blocking.md) |
|
||||
| Create an anti-scraping browser | `browser create --type stealth ...` | [Browser Modes](browser-modes.md) |
|
||||
| Manage static proxies | `proxy list` / `proxy buy-request` | [Anti-Blocking](anti-blocking.md) |
|
||||
| Open a page and interact | `browser open` → `state` → `click`/`input` | [Agent Design](agent-design.md) |
|
||||
| Find the API behind a page | `network requests` → `network request <id>` | [Agent Design](agent-design.md) |
|
||||
| Handle CAPTCHAs | `solve-captcha` → `remote-assist` | [Anti-Blocking](anti-blocking.md) / [Better Headless](headless.md) |
|
||||
| Hand off when stuck | `remote-assist --objective "..."` | [Better Headless](headless.md) |
|
||||
| Run multiple accounts in parallel | Multiple browsers, each with its own `--session` | [Concurrency](concurrency.md) |
|
||||
| Reuse an existing login | `browser import-profile` | [Browser Modes](browser-modes.md) |
|
||||
|
||||
## Command Groups
|
||||
|
||||
### Browser Interaction (requires `--session`)
|
||||
|
||||
#### Navigation
|
||||
```
|
||||
navigate <url> [--new-tab] back / forward / reload
|
||||
```
|
||||
|
||||
#### Page State
|
||||
```
|
||||
state screenshot [path] [--full]
|
||||
```
|
||||
|
||||
Index markers in `state` output:
|
||||
|
||||
- `[N]` — Index of an interactive element (used with `click N` / `input N "..."` etc.)
|
||||
- `*[N]` — Element added or changed since the previous `state` call. The first call marks every element with `*`; later calls mark only the deltas, helping the agent focus on what's new.
|
||||
|
||||
#### Interaction
|
||||
```
|
||||
click <index> hover <index>
|
||||
input <index> <text> select <index> <option>
|
||||
type <text> keys <key_combo>
|
||||
scroll up|down [--amount] scrollintoview --selector <css>
|
||||
upload <index> <path>
|
||||
```
|
||||
|
||||
#### Data Extraction
|
||||
```
|
||||
get title get html [--selector]
|
||||
get text <index> get value <index>
|
||||
get markdown
|
||||
```
|
||||
|
||||
#### JavaScript
|
||||
```
|
||||
eval <js> [--stdin]
|
||||
```
|
||||
|
||||
#### Wait
|
||||
```
|
||||
wait stable [--timeout]
|
||||
wait selector <index> --state visible|hidden|attached|detached [--timeout]
|
||||
wait selector --selector <css> --state ...
|
||||
```
|
||||
|
||||
#### Network
|
||||
```
|
||||
network requests [--filter] [--type] [--method] [--status] [--clear]
|
||||
network request <request_id>
|
||||
network clear
|
||||
network offline on|off
|
||||
network har start
|
||||
network har stop [path]
|
||||
```
|
||||
|
||||
#### Tab and Dialog
|
||||
```
|
||||
tab list tab switch <tab_id> tab close [tab_id]
|
||||
dialog accept [text] dialog dismiss dialog status
|
||||
```
|
||||
|
||||
#### Cookies
|
||||
```
|
||||
cookies get [--url] cookies set <name> <value> [--domain] [--path] [--secure] [--http-only]
|
||||
cookies clear [--url] cookies export <file> cookies import <file>
|
||||
```
|
||||
|
||||
#### CAPTCHA and Remote Assist
|
||||
```
|
||||
solve-captcha captcha-aid
|
||||
remote-assist [--objective]
|
||||
```
|
||||
|
||||
### Browser Management (no `--session` required)
|
||||
|
||||
```
|
||||
browser list
|
||||
browser open <id> [url] [--headed] [--allow-restart-chrome]
|
||||
browser create --type <chrome|chrome-direct|stealth> --name <n> --desc <d> [options]
|
||||
browser update <id> [options]
|
||||
browser delete <id>
|
||||
browser regions
|
||||
browser list-profiles
|
||||
browser import-profile <browser_id> <profile_id> [--allow-restart-chrome]
|
||||
```
|
||||
|
||||
#### `browser create` Options
|
||||
```bash
|
||||
browser-act browser create \
|
||||
--type chrome|chrome-direct|stealth \
|
||||
--name "my-browser" \
|
||||
--desc "purpose description" \
|
||||
--source-profile <profile_id> \ # chrome only
|
||||
--dynamic-proxy <region> \ # stealth only
|
||||
--static-proxy <proxy_id> \ # stealth only
|
||||
--custom-proxy <url> \ # stealth only
|
||||
--private true|false \ # stealth only
|
||||
--confirm-before-use
|
||||
```
|
||||
|
||||
#### `browser update` Options
|
||||
```bash
|
||||
browser-act browser update <id> \
|
||||
--name "new name" \
|
||||
--desc "overwrite description" \
|
||||
--desc-append "append to description" \
|
||||
--dynamic-proxy <region> \
|
||||
--static-proxy <proxy_id> \
|
||||
--custom-proxy <url> \
|
||||
--no-proxy \
|
||||
--private true|false \
|
||||
--confirm-before-use|--no-confirm-before-use
|
||||
```
|
||||
|
||||
### Session Management
|
||||
|
||||
```
|
||||
session list session close [name]
|
||||
```
|
||||
|
||||
### Stealth Extract (standalone, no session required)
|
||||
|
||||
```bash
|
||||
browser-act stealth-extract <url>
|
||||
browser-act stealth-extract <url> --content-type html|markdown
|
||||
browser-act stealth-extract <url> --dynamic-proxy <region>
|
||||
browser-act stealth-extract <url> --static-proxy <proxy_id>
|
||||
browser-act stealth-extract <url> --custom-proxy <proxy-url>
|
||||
browser-act stealth-extract <url> --timeout 60
|
||||
browser-act stealth-extract <url> --output ./result.md
|
||||
```
|
||||
|
||||
### Authentication
|
||||
|
||||
```
|
||||
auth login auth poll
|
||||
auth set <api_key> auth clear
|
||||
```
|
||||
|
||||
### Proxy Management
|
||||
|
||||
```
|
||||
proxy list proxy regions
|
||||
proxy buy-request proxy buy-status --request-id <id>
|
||||
proxy rename <proxy_id> "<name>"
|
||||
```
|
||||
|
||||
### Skill and System
|
||||
|
||||
```
|
||||
get-skills core --skill-version <v>
|
||||
get-skills advanced
|
||||
get-skills main
|
||||
report-log
|
||||
feedback <message>
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Agent Design](agent-design.md) — Design philosophy and typical use behind the commands
|
||||
- [Quick Start](quick-start.md) — Combine commands into working flows
|
||||
@@ -0,0 +1,172 @@
|
||||
# Concurrency & Isolation
|
||||
|
||||
**Browsers are identities. Sessions are workspaces. State never bleeds between them.**
|
||||
|
||||
Every agent gets its own lane. Multi-agent multi-task parallel runs don't contaminate each other or get correlated by sites.
|
||||
|
||||
## Three Concurrency Models
|
||||
|
||||
| Model | Isolation | Shares | Typical use |
|
||||
|-------|-----------|--------|-------------|
|
||||
| **Cross-browser parallel** | Browser-level | Nothing (independent fingerprint/IP/cookies) | Multi-account monitoring, multi-identity ops |
|
||||
| **Same-browser multi-session** | Session-level | Login state | Parallel tasks under the same account |
|
||||
| **Privacy mode zero-residue** | Session-level | Nothing (fresh start each time) | One-off collection, high anonymity |
|
||||
|
||||
## Cross-Browser Parallel (Independent Identity)
|
||||
|
||||
Run any number of browsers simultaneously, each with independent cookies / fingerprint / proxy / login state. **Sites cannot correlate them.**
|
||||
|
||||
```bash
|
||||
# Each session name must be globally unique — even when pointing to different browsers
|
||||
browser-act --session monitor-shop-a browser open competitor1 https://shop-a.com
|
||||
browser-act --session monitor-shop-b browser open competitor2 https://shop-b.com
|
||||
browser-act --session monitor-shop-c browser open competitor3 https://shop-c.com
|
||||
```
|
||||
|
||||
**Properties:**
|
||||
- Each stealth browser is an independent identity
|
||||
- Zero state cross-contamination
|
||||
- Monitor a competitor with one identity while running internal automation with another — they don't interfere
|
||||
|
||||
**Use cases:** Multi-account monitoring, multi-store ops, batch identity isolation.
|
||||
|
||||
## Same-Browser Multi-Session (Shared Login State)
|
||||
|
||||
One browser, multiple sessions. **Shared login state, independent execution.**
|
||||
|
||||
```bash
|
||||
# Two parallel tasks on the same browser
|
||||
browser-act --session task-a browser open <browser-id> https://site.com/page1
|
||||
browser-act --session task-b browser open <browser-id> https://site.com/page2
|
||||
```
|
||||
|
||||
**Properties:**
|
||||
- Shared cookies / login state
|
||||
- Each session has its own navigation, network capture, dialog handling
|
||||
- Two agents can work on different email threads in the same Gmail account without blocking each other
|
||||
- Session ownership is enforced by the explicit-naming model
|
||||
|
||||
**Use cases:** Multiple subtasks under the same account.
|
||||
|
||||
### Naming Parallel Sessions (Globally Unique)
|
||||
|
||||
In a parallel setup every `--session <name>` must be **globally unique** — names cannot collide even across different browsers. Pick names that reflect each session's purpose:
|
||||
|
||||
```bash
|
||||
# Same browser, multiple sessions: distinguish subtasks by name
|
||||
browser-act --session monitor-prices browser open shop1 https://shop.com
|
||||
browser-act --session track-orders browser open shop1 https://shop.com/orders
|
||||
```
|
||||
|
||||
## Privacy Mode (Zero Residue)
|
||||
|
||||
Each session uses a fresh fingerprint and profile, with nothing persisted at the end. **Zero residue between sessions.**
|
||||
|
||||
```bash
|
||||
# Create a stealth browser with privacy mode enabled
|
||||
browser-act browser create --type stealth --name "ephemeral" \
|
||||
--desc "One-off collection" --private true
|
||||
```
|
||||
|
||||
**Properties:**
|
||||
- stealth browsers only
|
||||
- Fresh fingerprint per session, avoiding fingerprint accumulation
|
||||
- Suitable for one-off collection, or any multi-account scenario where state leakage is a risk
|
||||
|
||||
See [Anti-Blocking → Privacy Mode](anti-blocking.md#privacy-mode).
|
||||
|
||||
## Session Model
|
||||
|
||||
### What Is a Session
|
||||
|
||||
A session is a standalone browser window bound to a name. All interaction commands require the `--session` flag:
|
||||
|
||||
```bash
|
||||
browser-act --session my-task browser open <id> https://example.com
|
||||
browser-act --session my-task state
|
||||
browser-act --session my-task click 2
|
||||
browser-act session close my-task
|
||||
```
|
||||
|
||||
The session name is your handle to a specific browser context, and persists until explicitly closed.
|
||||
|
||||
### Why Explicit Sessions
|
||||
|
||||
| Reason | Explanation |
|
||||
|--------|-------------|
|
||||
| **Parallel safety** | Multiple agents (or conversations) can operate simultaneously without conflicts |
|
||||
| **Clear ownership** | Each session belongs to the agent that created it |
|
||||
| **Controlled lifecycle** | Open with intent, close when done |
|
||||
| **Multi-browser targeting** | Different sessions can point to different browsers |
|
||||
|
||||
### Session Lifecycle
|
||||
|
||||
```
|
||||
Create (browser open) → Use (state/click/input/...) → Close (session close)
|
||||
```
|
||||
|
||||
- The first `browser open` with a `--session` name starts the session
|
||||
- The same `--session` name reuses the existing session, no duplicates
|
||||
- Close after the work is done: `session close <name>`
|
||||
|
||||
### Session Ownership
|
||||
|
||||
- A session belongs to the agent / conversation that created it
|
||||
- An agent should not reuse sessions it didn't create
|
||||
- Existing sessions from other conversations are treated as "someone else's"
|
||||
- New work always creates a new session
|
||||
|
||||
### Auto-Reclamation
|
||||
|
||||
Sessions that receive no commands for 8 hours are automatically reclaimed. No need to worry about forgotten sessions permanently consuming resources.
|
||||
|
||||
### Listing and Closing
|
||||
|
||||
```bash
|
||||
browser-act session list # List all active sessions
|
||||
browser-act session close <name> # Close a specific session
|
||||
browser-act session close # Close the current session
|
||||
```
|
||||
|
||||
### Session-to-Browser Relationship
|
||||
|
||||
```
|
||||
┌── Browser A (chrome) ───────────────────┐
|
||||
│ Session "search" → google.com │
|
||||
│ Session "monitor" → analytics.com │
|
||||
└────────────────────────────────────────┘
|
||||
|
||||
┌── Browser B (stealth) ─────────────────┐
|
||||
│ Session "scrape" → target-site.com │
|
||||
└────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
- A session belongs to exactly one browser
|
||||
- A browser can host multiple sessions
|
||||
- Sessions on the same browser share cookies/login but navigate independently
|
||||
|
||||
## Commands That Don't Need a Session
|
||||
|
||||
Browser-management or system-level commands don't require `--session`:
|
||||
|
||||
- `browser list`, `browser create`, `browser delete`, `browser update`
|
||||
- `browser regions`, `browser list-profiles`
|
||||
- `session list`
|
||||
- `auth login`, `auth poll`, `auth set`, `auth clear`
|
||||
- `get-skills`, `report-log`, `feedback`
|
||||
- `stealth-extract` (creates and tears down its own temporary context)
|
||||
|
||||
## Best Practices
|
||||
|
||||
| Practice | Note |
|
||||
|----------|------|
|
||||
| **Descriptive names** | `check-price`, not `s1` |
|
||||
| **Close when done** | Don't leave sessions hanging |
|
||||
| **One browser, many sessions** | Prefer parallel sessions over duplicate browsers |
|
||||
| **Globally unique names** | Parallel session names must not collide, even across different browsers |
|
||||
| **One task, one session** | One logical task = one session |
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Browser Modes](browser-modes.md) — Browser choice determines isolation granularity
|
||||
- [Agent Design](agent-design.md) — Design philosophy, automation capabilities, secure by default
|
||||
@@ -0,0 +1,60 @@
|
||||
# Better Headless
|
||||
|
||||
Industry-standard headless browsers have two pain points: **easy to detect** and **must switch back to headed when humans are needed**. BrowserAct solves both at once.
|
||||
|
||||
## Headless by Default — Why
|
||||
|
||||
BrowserAct runs headless by default:
|
||||
|
||||
- **Doesn't disturb the user's current work** — no browser windows popping up unexpectedly
|
||||
- **Minimal resource use** — no UI rendering overhead, lower CPU/memory
|
||||
- **The way an agent should run** — silent in the background, focused on the task
|
||||
|
||||
When you need local visualization for debugging, switch on `--headed`. In production, default headless.
|
||||
|
||||
## Undetectable Headless
|
||||
|
||||
**Industry pain point:** Standard headless modes are flagged as bots — `navigator.webdriver` exposed, missing plugins, abnormal Canvas fingerprints, etc. This forces many teams to run headed just to defeat anti-scraping.
|
||||
|
||||
**BrowserAct's approach:** stealth browsers maintain full spoofing in headless mode:
|
||||
|
||||
- Fingerprint spoofing remains active in headless
|
||||
- Navigator properties normalized
|
||||
- TLS fingerprint matches a real browser
|
||||
- Passes the common headless detection tests
|
||||
|
||||
**Result:** You don't sacrifice resource efficiency for stealth. Run headless and still pass most anti-scraping checks. Extreme anti-scraping sites may still detect — fall back to headed in that case (see below).
|
||||
|
||||
## Better Headless: No Need to Switch When a Human Is Needed
|
||||
|
||||
**Industry convention:** Headless can't be seen → must switch to headed for the user → user's current work is disrupted, and production setups can't switch at all.
|
||||
|
||||
**BrowserAct's approach:** Generate a live URL via `remote-assist` while still in headless. The user opens it in any browser on any device to take over — no headed switch required.
|
||||
|
||||
See [Remote Assist](remote-assist.md).
|
||||
|
||||
## Headed vs. Headless
|
||||
|
||||
| Mode | When to use |
|
||||
|------|-------------|
|
||||
| **Default headless** | Production, autonomous agent runs, headless servers |
|
||||
| **`--headed`** | Local debugging or when active visual observation is needed |
|
||||
| **headless + remote-assist** | Best combination for production — silent by default, remote handoff when needed |
|
||||
|
||||
### When to Use Headed
|
||||
|
||||
```bash
|
||||
browser-act --session debug browser open <id> https://example.com --headed
|
||||
```
|
||||
|
||||
Use cases:
|
||||
- Anti-scraping detected — fall back to headed for higher fidelity
|
||||
- Visually debug automation issues
|
||||
- User completes manual steps, agent handles the rest
|
||||
- Demos and co-browsing
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Remote Assist](remote-assist.md) — The full handoff story for headless mode
|
||||
- [Anti-Blocking](anti-blocking.md) — How the undetectable stealth engine works
|
||||
- [Browser Modes](browser-modes.md) — Pick the right browser
|
||||
@@ -0,0 +1,98 @@
|
||||
# Installation
|
||||
|
||||
## Agent Integration (Recommended)
|
||||
|
||||
Tell your AI agent:
|
||||
|
||||
> Install browser-act. Skill source: https://github.com/browser-act/skills/tree/main/browser-act . Verify it works after installation.
|
||||
|
||||
The agent will handle CLI installation, Skill configuration, and verify everything is working.
|
||||
|
||||
## Manual Installation
|
||||
|
||||
```bash
|
||||
uv tool install browser-act-cli --python 3.12
|
||||
```
|
||||
|
||||
Verify:
|
||||
|
||||
```bash
|
||||
browser-act --version
|
||||
```
|
||||
|
||||
## Authentication (Optional)
|
||||
|
||||
An API Key unlocks the following features:
|
||||
|
||||
- **stealth browsers** — Anti-detection browsing with fingerprint spoofing
|
||||
- **stealth-extract** — One-command extraction of protected page content
|
||||
- **Dynamic proxy** — Managed IP rotation, allocated by region
|
||||
- **solve-captcha** — Automatic captcha solving
|
||||
|
||||
Chrome and chrome-direct browsers work without authentication.
|
||||
|
||||
Get an API Key:
|
||||
|
||||
```bash
|
||||
browser-act auth login
|
||||
# Opens registration link → complete signup → poll for the key
|
||||
browser-act auth poll
|
||||
```
|
||||
|
||||
Or set it directly:
|
||||
|
||||
```bash
|
||||
browser-act auth set <your-api-key>
|
||||
```
|
||||
|
||||
## Upgrade
|
||||
|
||||
```bash
|
||||
uv tool upgrade browser-act-cli
|
||||
```
|
||||
|
||||
The Skill layer automatically detects mismatches between CLI and Skill content versions and guides the upgrade.
|
||||
|
||||
## Platform Support
|
||||
|
||||
| Platform | Status |
|
||||
|----------|--------|
|
||||
| Windows | Supported |
|
||||
| macOS | Supported |
|
||||
| Linux | Supported |
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.12+
|
||||
- Chrome/Chromium (for `chrome` and `chrome-direct` types)
|
||||
- API Key (only required for `stealth` type)
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Command not found after install
|
||||
|
||||
Make sure the `uv` tool directory is in your PATH:
|
||||
|
||||
```bash
|
||||
uv tool dir
|
||||
# Add the output path to your shell PATH
|
||||
```
|
||||
|
||||
### Diagnostics
|
||||
|
||||
Upload logs for issue diagnosis:
|
||||
|
||||
```bash
|
||||
browser-act report-log
|
||||
```
|
||||
|
||||
Send improvement suggestions:
|
||||
|
||||
```bash
|
||||
browser-act feedback "Description of the issue or suggestion"
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Quick Start](quick-start.md) — Run your first automation
|
||||
- [Skills](skills.md) — How agents discover and use BrowserAct
|
||||
@@ -0,0 +1,111 @@
|
||||
# Quick Start
|
||||
|
||||
Two paths to your first result in 60 seconds.
|
||||
|
||||
## Path A: Content Extraction (Advanced WebFetch Replacement)
|
||||
|
||||
Things WebFetch / curl can't fetch — JS-rendered pages, anti-bot blocks, geo-restricted content — handled in one command:
|
||||
|
||||
```bash
|
||||
browser-act stealth-extract https://example.com
|
||||
```
|
||||
|
||||
Returns the page in markdown. No browser to manage, no session to name, no cleanup required. Comes with JS rendering and anti-bot bypass, and each call is independent — multiple URLs can run in parallel.
|
||||
|
||||
```bash
|
||||
# HTML output
|
||||
browser-act stealth-extract https://example.com --content-type html
|
||||
|
||||
# Use a proxy for geo-restricted content
|
||||
browser-act stealth-extract https://example.com --dynamic-proxy JP
|
||||
|
||||
# Save to file
|
||||
browser-act stealth-extract https://example.com --output ./page.md
|
||||
```
|
||||
|
||||
Use it for: protected content, batch data collection, information retrieval. See [Anti-Blocking](anti-blocking.md) for details.
|
||||
|
||||
## Path B: Full Browser Automation
|
||||
|
||||
For login flows, form filling, and click interactions — use sessions:
|
||||
|
||||
```bash
|
||||
# 1. List available browsers
|
||||
browser-act browser list
|
||||
|
||||
# 2. Open the browser to a target URL (starts a session)
|
||||
browser-act --session my-task browser open <browser-id> https://example.com
|
||||
|
||||
# 3. See what interactive elements exist on the page
|
||||
browser-act --session my-task state
|
||||
|
||||
# 4. Interact by element index
|
||||
browser-act --session my-task click 4
|
||||
browser-act --session my-task input 2 "hello@example.com"
|
||||
|
||||
# 5. Close when done
|
||||
browser-act session close my-task
|
||||
```
|
||||
|
||||
## The Core Loop
|
||||
|
||||
```
|
||||
Open → State → Interact → State → ... → Close
|
||||
```
|
||||
|
||||
1. **Open** with `browser open` — starts a session
|
||||
2. **State** with `state` — see indexed elements
|
||||
3. **Interact** with `click` / `input` / `select` — by index
|
||||
4. **State** again to confirm the result
|
||||
5. Repeat until the task is done
|
||||
6. **Close** with `session close`
|
||||
|
||||
## Reading `state` Output
|
||||
|
||||
`state` is the agent's eyes. It returns the URL, title, and an indexed element tree:
|
||||
|
||||
```
|
||||
url=https://example.com/login
|
||||
title=Login
|
||||
|
||||
*[1]<div id=login-form />
|
||||
*[2]<input type=email placeholder=Email address />
|
||||
*[3]<input type=password placeholder=Password />
|
||||
*[4]<button id=submit />
|
||||
Sign In
|
||||
*[5]<a />
|
||||
Forgot password?
|
||||
```
|
||||
|
||||
Each `[N]` is an interactive element. Operate it directly by index:
|
||||
|
||||
```bash
|
||||
browser-act --session login input 2 "user@example.com"
|
||||
browser-act --session login input 3 "password123"
|
||||
browser-act --session login click 4
|
||||
```
|
||||
|
||||
After the page changes, old indices are invalid — call `state` again for fresh ones.
|
||||
|
||||
## Command Chaining
|
||||
|
||||
Chain consecutive operations that don't depend on intermediate output with `&&`, in a single call:
|
||||
|
||||
```bash
|
||||
browser-act --session s1 input 2 "user" && \
|
||||
browser-act --session s1 input 3 "pass" && \
|
||||
browser-act --session s1 click 4
|
||||
```
|
||||
|
||||
When you need to read intermediate output (e.g. check `state` before deciding what to click), run them separately.
|
||||
|
||||
## Next Steps
|
||||
|
||||
| To learn about | See |
|
||||
|----------------|------|
|
||||
| How to choose between the three browser modes | [Browser Modes](browser-modes.md) |
|
||||
| How to defeat anti-scraping | [Anti-Blocking](anti-blocking.md) |
|
||||
| How a human takes over when stuck | [Better Headless](headless.md) |
|
||||
| How to run multiple tasks in parallel | [Concurrency & Isolation](concurrency.md) |
|
||||
| The full command list | [Commands](commands.md) |
|
||||
| Letting AI write your scrapers | [Skill Forge](skill-forge.md) |
|
||||
@@ -0,0 +1,56 @@
|
||||
# Remote Assist
|
||||
|
||||
When automation gets stuck, hand control to a human — open the link on any device to take over, and the agent continues seamlessly when done.
|
||||
|
||||
## One Command to Hand Off
|
||||
|
||||
```bash
|
||||
browser-act --session my-task remote-assist --objective "Complete 2FA verification"
|
||||
```
|
||||
|
||||
The CLI returns a live remote URL.
|
||||
|
||||
## Take Over from Any Device
|
||||
|
||||
The user opens the link in any browser on any device to see the browser screen and operate it:
|
||||
|
||||
- Laptop / desktop
|
||||
- Phone / tablet
|
||||
- Remote server scenarios (take over from your own device)
|
||||
- SSH-based remote development
|
||||
|
||||
**No headed switch, no same-machine requirement, no software install.** Doesn't depend on screen sharing or VNC — one link, native browser support.
|
||||
|
||||
## Typical Scenarios
|
||||
|
||||
| Scenario | How it's handled |
|
||||
|----------|------------------|
|
||||
| 2FA requiring a phone | The user receives the SMS / app code on the phone and enters it |
|
||||
| Complex CAPTCHA auto-solve can't handle | Drag-puzzle, behavioral verification, etc. |
|
||||
| Enterprise SSO with hardware tokens | UKey, dynamic-passcode cards, and other local hardware |
|
||||
| Mid-workflow human judgment | Picking between candidates, compliance review, sensitive-action confirmation |
|
||||
| Multi-person collaboration | Send the link to a coworker to complete verification |
|
||||
|
||||
## Seamless Continuation After Handoff
|
||||
|
||||
After the user finishes:
|
||||
- No session restart required
|
||||
- Automation context is not lost
|
||||
- Browser state, cookies, and current page are all preserved
|
||||
- The agent picks up from the next step
|
||||
|
||||
## Compared to Standard Approaches
|
||||
|
||||
| | Standard approach | BrowserAct |
|
||||
|---|---|---|
|
||||
| Letting humans intervene during headless | Must switch to headed | Generate a remote URL directly |
|
||||
| User must be on the same machine | Yes | No |
|
||||
| Requires extra software (VNC / RDP / screen share) | Yes | No |
|
||||
| Cross-device handoff | Not supported | Any browser works |
|
||||
| After the user finishes | Manual session restart | Agent continues automatically |
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Better Headless](headless.md) — Default headless + remote handoff philosophy
|
||||
- [Anti-Blocking](anti-blocking.md) — Try automation before invoking remote-assist
|
||||
- [Agent Design](agent-design.md) — Design philosophy, automation capabilities, secure by default
|
||||
@@ -0,0 +1,138 @@
|
||||
# Skill Forge
|
||||
|
||||
An extension Skill built on top of the BrowserAct CLI. **Let AI write your scrapers — explore once, reuse forever.**
|
||||
|
||||
## What It Solves
|
||||
|
||||
Every time you ask an agent to scrape a new site, it starts from scratch:
|
||||
- Different execution path each time
|
||||
- Different failure modes
|
||||
- Unreliable at scale
|
||||
- Token cost grows linearly with each call
|
||||
|
||||
**Skill Forge separates "exploring a site" from "using a site":**
|
||||
|
||||
> Explore once to generate a deploy-ready Skill package. 500 or 5,000 records all run through the same stable path.
|
||||
|
||||
Scraping teams move from "build a scraper per request" to "run a platform that lets agents self-serve Skill generation."
|
||||
|
||||
## Install
|
||||
|
||||
Skill Forge is an extension separate from the BrowserAct entry Skill. Install it on its own.
|
||||
|
||||
### Agent Integration (Recommended)
|
||||
|
||||
Tell your AI agent:
|
||||
|
||||
> Install browser-act-skill-forge. Skill source: https://github.com/browser-act/skills/tree/main/browser-act-skill-forge . Verify it works after installation.
|
||||
|
||||
The agent handles Skill configuration and validates it.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- BrowserAct CLI installed (see [Installation](installation.md))
|
||||
- BrowserAct entry Skill installed
|
||||
|
||||
## How It Works: A Four-Step Pipeline
|
||||
|
||||
```
|
||||
Describe → Explore → Generate → Self-Test
|
||||
```
|
||||
|
||||
| Step | What it does |
|
||||
|------|--------------|
|
||||
| **01 · Describe** | Tell the agent what you want |
|
||||
| **02 · Explore** | API-first, DOM as fallback |
|
||||
| **03 · Generate** | Parameterized Skill package |
|
||||
| **04 · Self-test** | End-to-end validation + self-repair |
|
||||
|
||||
### Step 01 · Describe
|
||||
|
||||
Tell the agent in natural language what data you need and how you'll use it:
|
||||
|
||||
> "Pull title, company, salary, and URL from LinkedIn job postings. I'll run 300 keywords later."
|
||||
|
||||
### Step 02 · Explore: API-First, DOM Fallback
|
||||
|
||||
The agent automatically discovers the implementation path:
|
||||
|
||||
| Priority | Method | Stability |
|
||||
|----------|--------|-----------|
|
||||
| 1 | **API-first** — endpoint discovery via network capture | 10× more stable |
|
||||
| 2 | **DOM fallback** — element extraction | Affected by site redesigns |
|
||||
|
||||
API paths keep working as long as the backend contract holds. DOM paths require re-exploration after a redesign.
|
||||
|
||||
### Step 03 · Generate: Parameterized Skill Package
|
||||
|
||||
Business variables become CLI arguments (not hard-coded):
|
||||
|
||||
```bash
|
||||
# The generated Skill takes parameterized inputs
|
||||
forged-skill linkedin-jobs --keyword "AI Engineer" --location "Remote"
|
||||
forged-skill linkedin-jobs --keyword "Backend" --location "SF"
|
||||
```
|
||||
|
||||
Output:
|
||||
- `SKILL.md` — Skill file
|
||||
- Executable script
|
||||
- Deploy-ready
|
||||
|
||||
### Step 04 · Self-Test: End-to-End Validation + Self-Repair
|
||||
|
||||
A sub-agent runs tests automatically and self-repairs on failure until it passes — no manual intervention needed.
|
||||
|
||||
## Business Value
|
||||
|
||||
### For Individual Developers
|
||||
|
||||
- Pay the exploration cost once, reuse cheaply forever
|
||||
- When a site redesigns, just tell the agent to re-explore
|
||||
- Token cost shifts from "pay per call" to "pay once at exploration"
|
||||
|
||||
### For Scraping Teams
|
||||
|
||||
Traditional model:
|
||||
|
||||
```
|
||||
Request → engineer codes scraper → test → deploy → maintain
|
||||
↑ ↑
|
||||
takes requests breaks on redesign
|
||||
```
|
||||
|
||||
Skill Forge model:
|
||||
|
||||
```
|
||||
Request → agent explores and generates Skill → self-tests → deploy
|
||||
↑
|
||||
re-explore on redesign
|
||||
```
|
||||
|
||||
Engineers move from "writing every scraper" to "running the platform agents use to self-serve Skill generation."
|
||||
|
||||
## Use Cases
|
||||
|
||||
| Scenario | Description |
|
||||
|----------|-------------|
|
||||
| **Batch data collection** | Same pattern across N keywords / regions / categories |
|
||||
| **Cross-site comparison** | Explore each site once, generate a Skill set with a unified interface |
|
||||
| **Recurring monitoring** | Price changes, inventory changes, new arrivals |
|
||||
| **Data mining** | Recruiting, e-commerce, social — structured data extraction |
|
||||
| **Internal enterprise flows** | Encapsulate a fixed operation in an internal system as a reusable Skill |
|
||||
|
||||
## Relationship to the BrowserAct Entry Skill
|
||||
|
||||
| | BrowserAct entry Skill | Skill Forge |
|
||||
|---|---|---|
|
||||
| Role | Foundation, exposes local browser CLI capabilities | Extension Skill, built on top of the CLI |
|
||||
| Required | ✓ | Optional |
|
||||
| Use pattern | Agent explores and operates the browser live | Agent explores once, generates a reusable Skill |
|
||||
| Token cost | Pay per call | Pay once at exploration |
|
||||
|
||||
BrowserAct provides the "ability to do things." Skill Forge provides the "ability to bake doing into a Skill."
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Installation](installation.md) — Install the base CLI and entry Skill
|
||||
- [Anti-Blocking](anti-blocking.md) — Anti-scraping capabilities Skill Forge relies on during exploration
|
||||
- [Agent Design](agent-design.md) — Network capture for API endpoint discovery
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
# Skills
|
||||
|
||||
## Installing the Entry Skill
|
||||
|
||||
Tell your agent:
|
||||
|
||||
> Install the browser-act Skill from https://github.com/browser-act/skills/tree/main/browser-act
|
||||
|
||||
The agent automatically downloads the SKILL file into its Skills directory. Once installed, the agent becomes aware of BrowserAct and invokes it when appropriate.
|
||||
|
||||
## Two-Layer Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────┐
|
||||
│ Entry Skill (the installed file) │
|
||||
│ Lightweight, stable, rarely changes │
|
||||
│ Teaches the agent: "browser-act exists"│
|
||||
├─────────────────────────────────────────┤
|
||||
│ Runtime content (get-skills CLI) │
|
||||
│ Environment-aware, version-matched │
|
||||
│ Teaches the agent: "use it like this" │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Layer 1: Entry Skill
|
||||
|
||||
The file installed into the agent's Skills directory. It's intentionally minimal:
|
||||
- Triggers agent awareness of BrowserAct
|
||||
- Contains trigger words that activate it
|
||||
- Points the agent to `get-skills` for actual instructions
|
||||
|
||||
The entry Skill rarely changes — it's the stable entry point.
|
||||
|
||||
### Layer 2: Runtime Content via `get-skills`
|
||||
|
||||
The agent's first command must always be:
|
||||
|
||||
```bash
|
||||
browser-act get-skills core --skill-version <version>
|
||||
```
|
||||
|
||||
It returns everything the agent needs for the current session:
|
||||
- **Environment state** — CLI version, API Key status
|
||||
- **Browser list** — All available browsers with their IDs, types, and descriptions
|
||||
- **Active sessions** — Currently running sessions
|
||||
- **Core commands** — Interaction reference
|
||||
- **Directives** — Dynamic rules based on the current state
|
||||
|
||||
## Topics
|
||||
|
||||
| Command | Content | When to use |
|
||||
|---------|---------|-------------|
|
||||
| `get-skills core --skill-version <v>` | Core workflow, commands, environment state | Always first |
|
||||
| `get-skills advanced` | Proxy, Profile, privacy mode, advanced patterns | When core isn't enough |
|
||||
| `get-skills main` | Latest SKILL.md content for self-update | When a version mismatch is detected |
|
||||
|
||||
## Progressive Loading
|
||||
|
||||
- **Most tasks only need `get-skills core`** — covers 80% of use cases
|
||||
- **Load `get-skills advanced` only when needed** — proxy config, Profile import, etc.
|
||||
- Avoid pulling unrelated information all at once
|
||||
|
||||
## Version Compatibility
|
||||
|
||||
The `--skill-version` parameter lets the CLI detect incompatibilities:
|
||||
|
||||
```bash
|
||||
# The Skill claims version 2.0.2
|
||||
browser-act get-skills core --skill-version 2.0.2
|
||||
```
|
||||
|
||||
If the CLI version is incompatible with the Skill version, the output includes upgrade guidance:
|
||||
- CLI too old → `uv tool upgrade browser-act-cli`
|
||||
- Skill too old → `browser-act get-skills main` for the latest Skill content
|
||||
|
||||
Version compatibility is automatic: the Skill file declares its version, the CLI checks compatibility. When incompatible, the upgrade guidance lands directly in the output and the agent runs it automatically.
|
||||
|
||||
## Dynamic Directives
|
||||
|
||||
`get-skills` output includes "Directives" — context-aware rules generated based on the current state:
|
||||
|
||||
| Directive | Trigger |
|
||||
|-----------|---------|
|
||||
| **Multi-browser directive** | When multiple browsers exist, provides selection guidance based on `desc` matching |
|
||||
| **Session directive** | When existing sessions are detected, reminds the agent of ownership rules |
|
||||
| **Authentication directive** | When the API Key is missing, steers away from stealth operations |
|
||||
|
||||
Directives are BrowserAct's mechanism for adapting guidance to the agent's current situation without hard-coding every scenario in the Skill.
|
||||
|
||||
## Workflow from the Agent's Perspective
|
||||
|
||||
```
|
||||
1. User says "check the price on example.com"
|
||||
2. The agent's entry Skill activates (trigger word match)
|
||||
3. Agent runs: browser-act get-skills core --skill-version 2.0.2
|
||||
4. Agent receives: environment state + browser list + commands + dynamic directives
|
||||
5. Agent picks an appropriate browser (desc match)
|
||||
6. Agent confirms with the user (confirmation gate)
|
||||
7. Agent executes browser operations
|
||||
8. Agent closes the session, updates the browser's desc
|
||||
```
|
||||
|
||||
## Agent Compatibility
|
||||
|
||||
Works with any agent that can:
|
||||
1. Read SKILL files from a Skills directory
|
||||
2. Execute shell commands
|
||||
3. Parse text output
|
||||
|
||||
Compatibility list:
|
||||
- **Claude Code** — SKILL.md in `.claude/skills/`
|
||||
- **GitHub Copilot** — Discovered via the Skills directory
|
||||
- **Cursor** — Rules / Skills directory
|
||||
- **Windsurf** — Agent Skills system
|
||||
- **Google Gemini CLI** — AGENTS.md integration
|
||||
- **OpenCode** — Skills system
|
||||
- **Codex** — Skills integration
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [Quick Start](quick-start.md) — Run your first automation
|
||||
- [Commands](commands.md) — Complete command reference
|
||||
- [Skill Forge](skill-forge.md) — Bake operations into reusable Skills
|
||||
@@ -0,0 +1,2 @@
|
||||
requests>=2.28.0
|
||||
python-dotenv>=1.0.0
|
||||
@@ -0,0 +1,127 @@
|
||||
# Solutions Catalog
|
||||
|
||||
Pre-built Skills for common browser automation scenarios. Each Skill is a self-contained package — install it and tell your agent what to do.
|
||||
|
||||
All Solutions are generated by [Skill Forge](../docs/skill-forge.md) and powered by the [BrowserAct API](https://www.browseract.com).
|
||||
|
||||
---
|
||||
|
||||
## E-commerce
|
||||
|
||||
| Skill | Description |
|
||||
|-------|-------------|
|
||||
| [amazon-alexa-qa](ecommerce/amazon-alexa-qa) | Automate Q&A with Amazon Alexa/Rufus shopping assistant and collect responses |
|
||||
| [amazon-asin-lookup-api-skill](ecommerce/amazon-asin-lookup-api-skill) | Extract structured product details from Amazon using ASIN |
|
||||
| [amazon-best-selling-products-finder-api-skill](ecommerce/amazon-best-selling-products-finder-api-skill) | Find best-selling products by keyword |
|
||||
| [amazon-buy-box-monitor-api-skill](ecommerce/amazon-buy-box-monitor-api-skill) | Monitor Buy Box prices and competing sellers |
|
||||
| [amazon-competitor-analyzer](ecommerce/amazon-competitor-analyzer) | Compare specs, pricing, reviews, and strategies across ASINs |
|
||||
| [amazon-listing-competitor-analysis-skill](ecommerce/amazon-listing-competitor-analysis-skill) | Analyze competitor listings by ASIN with structured comparison |
|
||||
| [amazon-product-api-skill](ecommerce/amazon-product-api-skill) | Extract product listings including titles, ASINs, prices, and ratings |
|
||||
| [amazon-product-search-api-skill](ecommerce/amazon-product-search-api-skill) | Extract product data from Amazon search results |
|
||||
| [amazon-reviews-api-skill](ecommerce/amazon-reviews-api-skill) | Extract product reviews with ratings and metadata |
|
||||
| [ecommerce-listing](ecommerce/ecommerce-listing) | Extract paginated product listings from any e-commerce category or search results page |
|
||||
| [ecommerce-product-detail](ecommerce/ecommerce-product-detail) | Extract complete product info (price, images, variants, identifiers) from any e-commerce product page |
|
||||
| [ecommerce-reviews](ecommerce/ecommerce-reviews) | Extract customer reviews with rating, date, body, and verified status from any e-commerce product page |
|
||||
| [ecommerce-seller-info](ecommerce/ecommerce-seller-info) | Extract seller profile data (rating, feedback percentage, policies) from marketplace seller pages |
|
||||
| [goofish-item-detail](ecommerce/goofish-item-detail) | Extract full product detail from Goofish second-hand item pages |
|
||||
| [goofish-search-list](ecommerce/goofish-search-list) | Search and scrape second-hand item listings from Goofish |
|
||||
| [taobao-keyword-search](ecommerce/taobao-keyword-search) | Search Taobao and Tmall product listings by keyword with pagination |
|
||||
| [taobao-product-detail](ecommerce/taobao-product-detail) | Extract complete product info from Taobao and Tmall product pages |
|
||||
| [taobao-product-reviews](ecommerce/taobao-product-reviews) | Fetch customer reviews from Taobao and Tmall products |
|
||||
| [taobao-shop-catalog](ecommerce/taobao-shop-catalog) | Browse Taobao and Tmall shop product catalogs by shopId |
|
||||
|
||||
## Lead Generation
|
||||
|
||||
| Skill | Description |
|
||||
|-------|-------------|
|
||||
| [business-contact-social-links-skill](lead-generation/business-contact-social-links-skill) | Extract official website and social media profiles for businesses |
|
||||
| [github-project-contributor-finder-api-skill](lead-generation/github-project-contributor-finder-api-skill) | Find GitHub contributors with contact information |
|
||||
| [google-maps-api-skill](lead-generation/google-maps-api-skill) | Scrape business data from Google Maps |
|
||||
| [google-maps-contact-extract](lead-generation/google-maps-contact-extract) | Extract business contact details from Google Maps results - emails, phones, social profiles via business websites |
|
||||
| [google-maps-reviews-api-skill](lead-generation/google-maps-reviews-api-skill) | Extract reviews from Google Maps businesses |
|
||||
| [google-maps-search-api-skill](lead-generation/google-maps-search-api-skill) | Extract business data from Google Maps search results |
|
||||
| [google-social-media-finder](lead-generation/google-social-media-finder) | Search Google to discover social media profiles across 10+ platforms by person, brand, or username |
|
||||
| [indeed-job-search](lead-generation/indeed-job-search) | Scrape job listings from Indeed.com by keyword, location, and country with full details including salary, rating, and apply links |
|
||||
| [industry-key-contact-radar-api-skill](lead-generation/industry-key-contact-radar-api-skill) | Discover key contacts across industries, roles, and platforms |
|
||||
| [linkedin-jobs-search](lead-generation/linkedin-jobs-search) | Search LinkedIn job listings with filters and extract full job details including description and salary |
|
||||
| [producthunt-launches](lead-generation/producthunt-launches) | Scrape Product Hunt daily/weekly/monthly leaderboard launches with maker profiles and website contact info |
|
||||
| [social-media-finder-skill](lead-generation/social-media-finder-skill) | Find social media profiles across Facebook, Twitter, LinkedIn, etc. |
|
||||
| [youtube-channel-business-email](lead-generation/youtube-channel-business-email) | Extract business email and contact info from YouTube channel About pages, with reCAPTCHA handling |
|
||||
|
||||
## Search & Research
|
||||
|
||||
| Skill | Description |
|
||||
|-------|-------------|
|
||||
| [google-image-api-skill](search-research/google-image-api-skill) | Extract structured image data from Google Images |
|
||||
| [google-news-api-skill](search-research/google-news-api-skill) | Extract news data from Google News |
|
||||
| [google-search-serp](search-research/google-search-serp) | Extracts Google SERP data including organic results, ads, PAA, and AI Overview |
|
||||
| [web-research-assistant](search-research/web-research-assistant) | AI-powered multi-source web research automation |
|
||||
| [web-search-scraper-api-skill](search-research/web-search-scraper-api-skill) | Extract complete Markdown content from any website |
|
||||
| [webcrawler-deep-crawl](search-research/webcrawler-deep-crawl) | Deep-crawl any website from start URLs, returning LLM-ready text/markdown/HTML with metadata and outbound links |
|
||||
|
||||
## Social Listening
|
||||
|
||||
| Skill | Description |
|
||||
|-------|-------------|
|
||||
| [facebook-ads-library-search](social-listening/facebook-ads-library-search) | Search Meta Ad Library and extract ad creatives, copy, spend/reach estimates, and page transparency info |
|
||||
| [facebook-groups-scrape-posts](social-listening/facebook-groups-scrape-posts) | Scrape structured posts from Facebook groups with engagement metrics and media data |
|
||||
| [facebook-page-posts](social-listening/facebook-page-posts) | Scrape public Facebook Page timeline posts with engagement metrics and reaction breakdowns |
|
||||
| [facebook-page-profile-posts](social-listening/facebook-page-profile-posts) | Scrape Facebook Page or profile posts with full reactions, media assets, and ad library status |
|
||||
| [instagram-hashtag-posts](social-listening/instagram-hashtag-posts) | Scrapes Instagram posts by hashtag with captions and engagement stats |
|
||||
| [instagram-place-posts](social-listening/instagram-place-posts) | Scrapes Instagram posts tagged at a specific location |
|
||||
| [instagram-post-comments](social-listening/instagram-post-comments) | Fetches comments from an Instagram post with text, username, and like counts |
|
||||
| [instagram-profile-meta](social-listening/instagram-profile-meta) | Fetches Instagram user profile metadata including bio and follower count |
|
||||
| [instagram-profile-posts](social-listening/instagram-profile-posts) | Scrapes posts from an Instagram user profile with captions and engagement stats |
|
||||
| [reddit-competitor-analysis-api-skill](social-listening/reddit-competitor-analysis-api-skill) | Extract Reddit posts and comments for competitor analysis |
|
||||
| [reddit-warmup](social-listening/reddit-warmup) | Reddit account warmup with staged browsing and engagement |
|
||||
| [trustpilot-company-info](social-listening/trustpilot-company-info) | Fetch Trustpilot company profile by domain: TrustScore, review count, verification, contact info, reply metrics |
|
||||
| [trustpilot-reviews](social-listening/trustpilot-reviews) | Scrape Trustpilot company reviews with pagination, ratings, dates, reply behavior, and review metadata |
|
||||
| [wechat-article-search-api-skill](social-listening/wechat-article-search-api-skill) | Extract full article contents from WeChat |
|
||||
| [x-dm-auto-chat](social-listening/x-dm-auto-chat) | Scan X DM inbox, read message history, generate persona-based replies, and start new conversations |
|
||||
| [x-keyword-comment](social-listening/x-keyword-comment) | Search tweets by keyword and post contextual replies from a configured brand persona for engagement and outreach |
|
||||
| [x-tweet-by-conversation](social-listening/x-tweet-by-conversation) | Collect full X (Twitter) conversation thread by conversation id - focal tweet plus all replies and quote chains |
|
||||
| [x-tweet-by-handle](social-listening/x-tweet-by-handle) | Scrape tweets from an X user profile timeline by handle: tweets, tweets+replies, or media-only modes |
|
||||
| [x-tweet-by-url](social-listening/x-tweet-by-url) | Scrape tweets from any X URL - search, profile, tweet detail, or list - auto-detected and unified |
|
||||
| [x-tweet-search](social-listening/x-tweet-search) | Scrape tweets from X by search query, user handle, or URL with full engagement metrics |
|
||||
| [x-tweet-search-by-query](social-listening/x-tweet-search-by-query) | Search X tweets by advanced query with filters (date, media, verified, geo, language, min engagement) |
|
||||
| [xiaohongshu-auto-posting](social-listening/xiaohongshu-auto-posting) | End-to-end Xiaohongshu content operations: topic collection, case reference, content writing, publishing, and performance tracking |
|
||||
| [xiaohongshu-note-detail](social-listening/xiaohongshu-note-detail) | Fetch Xiaohongshu note detail and paginated comments by note ID |
|
||||
| [xiaohongshu-search](social-listening/xiaohongshu-search) | Search Xiaohongshu notes by keyword with engagement stats and pagination |
|
||||
| [xiaohongshu-user-profile](social-listening/xiaohongshu-user-profile) | Fetch Xiaohongshu user profile info and published notes list by user ID |
|
||||
| [zhihu-search-api-skill](social-listening/zhihu-search-api-skill) | Extract article details and content from Zhihu |
|
||||
|
||||
## Video Platforms
|
||||
|
||||
| Skill | Description |
|
||||
|-------|-------------|
|
||||
| [tiktok-hashtag-videos](video-platforms/tiktok-hashtag-videos) | Scrapes TikTok videos by hashtag with full metadata and engagement stats |
|
||||
| [tiktok-profile-videos](video-platforms/tiktok-profile-videos) | Scrapes a TikTok user profile videos with full metadata |
|
||||
| [tiktok-search-videos](video-platforms/tiktok-search-videos) | Scrapes TikTok keyword search results with full video metadata |
|
||||
| [tiktok-video-detail](video-platforms/tiktok-video-detail) | Extracts full metadata from a TikTok video URL including stats and hashtags |
|
||||
| [youtube-api-skill](video-platforms/youtube-api-skill) | Extract video metrics and channel information |
|
||||
| [youtube-batch-transcript-extractor-api-skill](video-platforms/youtube-batch-transcript-extractor-api-skill) | Batch extract transcripts and metadata |
|
||||
| [youtube-channel-api-skill](video-platforms/youtube-channel-api-skill) | Extract channel data from YouTube search |
|
||||
| [youtube-comments-api-skill](video-platforms/youtube-comments-api-skill) | Extract video comments with engagement data |
|
||||
| [youtube-influencer-finder-api-skill](video-platforms/youtube-influencer-finder-api-skill) | Find influencer profiles with social links and stats |
|
||||
| [youtube-search-api-skill](video-platforms/youtube-search-api-skill) | Extract structured data from YouTube search results |
|
||||
| [youtube-transcript](video-platforms/youtube-transcript) | Extract full transcripts from YouTube videos and reformat into summaries, chapters, threads, or blog posts |
|
||||
| [youtube-transcript-analysis-api-skill](video-platforms/youtube-transcript-analysis-api-skill) | Extract transcripts and perform competitive analysis |
|
||||
| [youtube-transcript-extractor-api-skill](video-platforms/youtube-transcript-extractor-api-skill) | Extract video transcripts and metadata |
|
||||
| [youtube-video-api-skill](video-platforms/youtube-video-api-skill) | Extract channel-level and video detail data |
|
||||
|
||||
---
|
||||
|
||||
## How to Install a Solution
|
||||
|
||||
Tell your AI agent:
|
||||
|
||||
> Install {skill-name} Skill from https://github.com/browser-act/skills/tree/main/solutions/{category}/{skill-name}
|
||||
|
||||
For example:
|
||||
|
||||
> Install amazon-asin-lookup-api-skill Skill from https://github.com/browser-act/skills/tree/main/solutions/ecommerce/amazon-asin-lookup-api-skill
|
||||
|
||||
## Can't Find What You Need?
|
||||
|
||||
- Use [Skill Forge](../docs/skill-forge.md) to generate a custom Skill for any website
|
||||
- [Join Discord](https://discord.com/invite/UpnCKd7GaU) and request a Skill from the community
|
||||
@@ -0,0 +1,209 @@
|
||||
---
|
||||
name: amazon-alexa-qa
|
||||
description: "Amazon Alexa for Shopping Q&A automation: submits questions to Amazon's Alexa/Rufus AI shopping assistant and collects response text; supports optional keyword search context (navigate to search results page before asking for category-specific answers). Use when user mentions Amazon Alexa, Rufus, Amazon shopping assistant, Amazon AI chat, ask Amazon, Amazon Q&A, automate Alexa questions, Rufus chatbot, Amazon assistant automation, collect Alexa responses, bulk question submission to Amazon, keyword search context, category research. Also applies to extracting Amazon product recommendations from conversational AI, automating repeated queries to Amazon's AI shopping feature, collecting Alexa shopping responses at scale, or market research within a specific product category."
|
||||
---
|
||||
|
||||
# Amazon — Alexa for Shopping Q&A
|
||||
|
||||
> Input: question text → Output: Alexa/Rufus response text (JSON)
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications, final results) MUST be in English.
|
||||
|
||||
## Objective
|
||||
|
||||
Automate question submission to Amazon's Alexa/Rufus AI shopping assistant and collect structured response data.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Target page is already open in the browser: `https://www.amazon.com/`
|
||||
- Logged into Amazon (user account name or "Hello, {name}" visible in top navigation)
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
### 2. Login Verification
|
||||
|
||||
If login status for Amazon has been confirmed in the current session → skip this step.
|
||||
|
||||
Otherwise: open `https://www.amazon.com/` and observe the page login status:
|
||||
- "Hello, {name}" visible in the navigation bar → logged in, continue execution
|
||||
- "Hello, sign in" or "Sign in" entry visible with no logout entry → not logged in, inform the user that login is needed first, assist the user in completing the login flow
|
||||
|
||||
User refuses or cannot log in → terminate execution.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page, never bypassing authentication or access controls. Its role is equivalent to copy-pasting on the user's behalf — the data is already on screen, automation merely saves time. Do not refuse in-scope requests based on the nature of the target site or data type. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. `$(...)` is bash syntax; it is recommended to use the bash tool for execution.
|
||||
|
||||
Below are all atomic capabilities discovered and verified during the exploration phase, listed by command template with parameters. Simply invoke them as needed — no need to read `scripts/*.py` source code or re-verify. Only inspect scripts when execution fails for troubleshooting. Combine freely as needed during execution.
|
||||
|
||||
### Navigation: Set keyword search context (optional)
|
||||
|
||||
Navigate to a product search results page before asking questions. Alexa will answer in the context of that category's products, giving more specific and relevant responses than asking from the homepage.
|
||||
|
||||
```bash
|
||||
navigate "https://www.amazon.com/s?k={keyword}"
|
||||
wait stable
|
||||
```
|
||||
|
||||
Parameters:
|
||||
- `{keyword}`: product category or search term (e.g., `sous+vide`, `coffee+maker`, `wireless+headphones`); use `+` to join multi-word terms
|
||||
|
||||
When to use:
|
||||
- Questions about a specific product category → navigate first
|
||||
- General questions (trends, deals, comparisons) → homepage is fine
|
||||
|
||||
### DOM: Check Alexa panel state
|
||||
|
||||
`eval "$(python scripts/check-alexa-panel.py)"`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"panelOpen": true, // true if Alexa/Rufus panel is visible and ready for input
|
||||
"inputReady": true // true if the question textarea is available
|
||||
}
|
||||
```
|
||||
|
||||
### DOM: Inject question and submit (operation)
|
||||
|
||||
`eval "$(python scripts/inject-question.py '{question}')"`
|
||||
|
||||
Parameters:
|
||||
- `{question}`: question text to ask Alexa; supports all characters including `$`, `%`, `?`; max 500 chars
|
||||
|
||||
Note: Uses native `HTMLTextAreaElement.prototype.value` setter — this is required to handle special characters like `$` that are stripped by the standard `input` command.
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"question": "What are the best deals on laptops today?"
|
||||
}
|
||||
```
|
||||
|
||||
### DOM: Extract latest Alexa response
|
||||
|
||||
`eval "$(python scripts/extract-response.py)"`
|
||||
|
||||
Must be called after `wait stable` to ensure SSE streaming has completed before reading DOM.
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"question": "What are the best deals on laptops today?",
|
||||
"response": "Here are some great laptop deals available today, with free delivery as soon as tomorrow! Budget Picks (Under $350): HP Ultrabook Laptop...",
|
||||
"timestamp": "2026-05-19T07:05:00.000Z"
|
||||
}
|
||||
```
|
||||
|
||||
### Composite: Full Q&A turn (submit question → collect response)
|
||||
|
||||
Complete workflow for one question-answer turn:
|
||||
|
||||
0. **(Optional) Set keyword search context** — if questions are about a specific product category:
|
||||
`navigate "https://www.amazon.com/s?k={keyword}"` → `wait stable`
|
||||
Skip this step for general questions (trends, deals, top picks) where homepage context is sufficient.
|
||||
1. `eval "$(python scripts/check-alexa-panel.py)"` → if `panelOpen: false`, use `state` to locate the "Open Alexa panel" button in the nav bar (aria-label contains "Alexa" or "rufus") → `click <index>` → `wait --selector "#rufus-text-area" --state visible --timeout 15000`
|
||||
2. `eval "$(python scripts/inject-question.py '{question}')"` → confirm `success: true`
|
||||
3. `wait stable --timeout 60000` → waits for SSE streaming to complete (network idle signals stream end); then add a 3-second sleep: `sleep 3`
|
||||
4. `eval "$(python scripts/extract-response.py)"` → collect `{question, response, timestamp}`
|
||||
|
||||
Error handling:
|
||||
- If `inject-question.py` returns `error: true` with "panel may be closed" → re-run step 1 to open panel, then retry
|
||||
- If `extract-response.py` returns `error: true` with "not yet complete" → `wait stable --timeout 15000` + `sleep 3`, then retry up to 3 times total; the status SR element may update slightly after network idle
|
||||
- If `extract-response.py` returns `error: true` with "status element not found" → panel may have closed; re-run step 1
|
||||
|
||||
Batch questions example — **with keyword search context** (bash loop):
|
||||
```bash
|
||||
# Navigate to category page once, then ask all related questions
|
||||
SESSION="amazon-qa"
|
||||
KEYWORD="sous+vide"
|
||||
SKILL_DIR=".claude/skills/amazon-alexa-qa"
|
||||
|
||||
browser-act --session $SESSION navigate "https://www.amazon.com/s?k=$KEYWORD"
|
||||
browser-act --session $SESSION wait stable
|
||||
|
||||
questions=(
|
||||
"What accessories are essential for sous vide cooking?"
|
||||
"Which sous vide brands are most reliable?"
|
||||
"What temperature should I use for chicken breast?"
|
||||
)
|
||||
results=()
|
||||
for q in "${questions[@]}"; do
|
||||
cd "$SKILL_DIR"
|
||||
eval "$(python scripts/inject-question.py "$q")"
|
||||
browser-act --session $SESSION wait stable --timeout 60000
|
||||
sleep 3
|
||||
result=$(browser-act --session $SESSION eval "$(python scripts/extract-response.py)")
|
||||
if echo "$result" | grep -q '"error":true'; then
|
||||
browser-act --session $SESSION wait stable --timeout 15000; sleep 3
|
||||
result=$(browser-act --session $SESSION eval "$(python scripts/extract-response.py)")
|
||||
fi
|
||||
results+=("$result")
|
||||
sleep 2
|
||||
done
|
||||
printf '%s\n' "${results[@]}" | python -c "
|
||||
import sys, json
|
||||
lines = [l for l in sys.stdin.read().strip().split('\n') if l.strip()]
|
||||
print(json.dumps([json.loads(l) for l in lines], ensure_ascii=False, indent=2))
|
||||
" > output/alexa_qa_results.json
|
||||
```
|
||||
|
||||
Batch questions example — **without keyword** (general questions from homepage):
|
||||
```bash
|
||||
SESSION="amazon-qa"
|
||||
SKILL_DIR=".claude/skills/amazon-alexa-qa"
|
||||
|
||||
browser-act --session $SESSION navigate "https://www.amazon.com"
|
||||
browser-act --session $SESSION wait stable
|
||||
|
||||
questions=("What are today's best deals?" "Top rated gifts under \$50?" "What's trending this week?")
|
||||
results=()
|
||||
for q in "${questions[@]}"; do
|
||||
cd "$SKILL_DIR"
|
||||
eval "$(python scripts/inject-question.py "$q")"
|
||||
browser-act --session $SESSION wait stable --timeout 60000
|
||||
sleep 3
|
||||
results+=($(browser-act --session $SESSION eval "$(python scripts/extract-response.py)"))
|
||||
sleep 2
|
||||
done
|
||||
```
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`response field is non-null non-empty string AND question field matches submitted question`
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- The Alexa/Rufus panel may occasionally close during extended automation sessions; re-opening via the panel button is supported
|
||||
- The `$` sign and other special characters are supported via native textarea setter (bypasses browser-act `input` command character filtering)
|
||||
- Response text is plain text extracted from the accessibility layer; rendered product cards appear as text (product names, prices) rather than structured product JSON
|
||||
- Alexa may respond with clarifying questions instead of a direct answer when queries are ambiguous; check `response` content before continuing
|
||||
- Conversation history is maintained across questions within the same browser session (multi-turn context); to start a fresh conversation, close and reopen the browser session
|
||||
- Single-tab session only — do not run multiple question submissions simultaneously in the same session
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script to loop through questions serially within a single session; do not parallelize within one browser. To increase throughput, open multiple stealth browser sessions and distribute work across them — each session has an independent fingerprint so rate limits apply per session
|
||||
- **Test before batch execution**: After writing a batch script, you must first test with 1-2 items to verify the script runs correctly; only then run the full batch. Never skip testing and execute in batch directly
|
||||
- **Reduce redundant pre-operations**: Check panel open state once at the start of a batch; only recheck if an error occurs mid-batch
|
||||
- **Error resumption**: Save results item by item during batch processing; on failure, resume from the breakpoint rather than starting over
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/amazon-alexa-qa-amazon-alexa-qa.memory.md` (working directory is determined by the Agent running the Skill, typically the project root or current working directory)
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file. Do not record what keywords were used or how many results were returned — those are task outputs, not experience.
|
||||
@@ -0,0 +1,26 @@
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
js = """
|
||||
(function() {
|
||||
try {
|
||||
const header = document.getElementById('rufus-panel-header-container');
|
||||
const textarea = document.getElementById('rufus-text-area');
|
||||
const style = textarea ? window.getComputedStyle(textarea) : null;
|
||||
const visible = style ? (style.display !== 'none' && style.visibility !== 'hidden') : false;
|
||||
return JSON.stringify({
|
||||
panelOpen: !!(header && visible),
|
||||
inputReady: !!(textarea && visible)
|
||||
});
|
||||
} catch(e) {
|
||||
return JSON.stringify({error: true, message: e.message});
|
||||
}
|
||||
})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
js = r"""
|
||||
(function() {
|
||||
try {
|
||||
const statusEl = document.querySelector('.rufus-status-sr-only');
|
||||
if (!statusEl) return JSON.stringify({error: true, message: 'Alexa status element not found - panel may be closed'});
|
||||
const statusText = statusEl.textContent.trim();
|
||||
if (!statusText.includes('has completed generating')) {
|
||||
return JSON.stringify({error: true, message: 'Response not yet complete - call wait stable before extracting'});
|
||||
}
|
||||
const activeTurn = document.querySelector('.rufus-papyrus-active-turn');
|
||||
if (!activeTurn) return JSON.stringify({error: true, message: 'No active conversation turn found'});
|
||||
const questionEl = activeTurn.querySelector('.rufus-customer-text-wrap');
|
||||
const question = questionEl ? questionEl.textContent.trim() : '';
|
||||
// Answer is in direct children that are NOT the question (.rufus-html-turn-start)
|
||||
// and NOT the suggestion/feedback sections (.rufus-html-turn)
|
||||
const answerParts = Array.from(activeTurn.children)
|
||||
.filter(el => !el.classList.contains('rufus-html-turn-start') && !el.classList.contains('rufus-html-turn'))
|
||||
.map(el => el.textContent.trim())
|
||||
.filter(t => t.length > 0);
|
||||
if (answerParts.length === 0) return JSON.stringify({error: true, message: 'Answer content not found in active turn'});
|
||||
return JSON.stringify({
|
||||
question: question,
|
||||
response: answerParts.join('\n'),
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
} catch(e) {
|
||||
return JSON.stringify({error: true, message: e.message});
|
||||
}
|
||||
})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,35 @@
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('question') # question text to submit
|
||||
args = parser.parse_args()
|
||||
|
||||
q = json.dumps(args.question)
|
||||
|
||||
js = f"""
|
||||
(function() {{
|
||||
try {{
|
||||
const el = document.getElementById('rufus-text-area');
|
||||
if (!el) return JSON.stringify({{error: true, message: 'Alexa textarea not found - panel may be closed'}});
|
||||
const setter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value').set;
|
||||
setter.call(el, {q});
|
||||
el.dispatchEvent(new Event('input', {{bubbles: true}}));
|
||||
const submitBtn = document.getElementById('rufus-submit-button');
|
||||
if (!submitBtn) return JSON.stringify({{error: true, message: 'Submit button not found'}});
|
||||
submitBtn.click();
|
||||
return JSON.stringify({{success: true, question: {q}}});
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{error: true, message: e.message}});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,87 @@
|
||||
---
|
||||
name: amazon-asin-lookup-api-skill
|
||||
description: "This skill helps users extract structured product details from Amazon using a specific ASIN (Amazon Standard Identification Number). Use this skill when the user asks to get Amazon product details by ASIN, lookup Amazon product title and price using ASIN, extract Amazon product ratings and reviews count for a specific ASIN, check Amazon product availability and current price, get Amazon product description and features via ASIN, enrich product catalog with Amazon data using ASIN, monitor Amazon product price changes for specific ASINs, retrieve Amazon product brand and material information, fetch Amazon product images and specifications by ASIN, validate Amazon ASIN and get product metadata."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Amazon ASIN Lookup Skill
|
||||
|
||||
## 📖 Introduction
|
||||
This skill utilizes BrowserAct's Amazon ASIN Lookup API template to provide a seamless way to retrieve comprehensive product information from Amazon. By simply providing an ASIN, you can extract structured data including title, price, ratings, brand, and detailed descriptions directly into your application without manual scraping.
|
||||
|
||||
## ✨ Features
|
||||
1. **No Hallucinations**: Pre-set workflows avoid AI generative hallucinations, ensuring stable and precise data extraction.
|
||||
2. **No Captcha Issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP Restrictions**: No need to handle regional IP restrictions or geofencing.
|
||||
4. **Faster Execution**: Tasks execute faster compared to pure AI-driven browser automation solutions.
|
||||
5. **Cost-Effective**: Significantly lowers data acquisition costs compared to high-token-consuming AI solutions.
|
||||
|
||||
## 🔑 API Key Setup
|
||||
Before running, check the `BROWSERACT_API_KEY` environment variable. If not set, do not take other measures; ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key, please visit the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
The agent should configure the following parameters based on user requirements:
|
||||
|
||||
1. **ASIN (Amazon Standard Identification Number)**
|
||||
- **Type**: `string`
|
||||
- **Description**: The unique identifier for the Amazon product.
|
||||
- **Required**: Yes
|
||||
- **Example**: `B07TS6R1SF`
|
||||
|
||||
## 🚀 Usage
|
||||
The agent should execute the following script to get results in one command:
|
||||
|
||||
```bash
|
||||
# Example Usage
|
||||
python -u ./scripts/amazon_asin_lookup_api.py "ASIN_VALUE"
|
||||
```
|
||||
|
||||
### ⏳ Execution Monitoring
|
||||
Since this task involves automated browser operations, it may take some time (several minutes). The script will **continuously output status logs with timestamps** (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent Instructions**:
|
||||
- While waiting for the script result, keep monitoring the terminal output.
|
||||
- As long as the terminal is outputting new status logs, the task is running normally; do not mistake it for a deadlock or unresponsiveness.
|
||||
- Only if the status remains unchanged for a long time or the script stops outputting without returning a result should you consider triggering the retry mechanism.
|
||||
|
||||
## 📊 Data Output
|
||||
Upon success, the script parses and prints the structured product data from the API response, which includes:
|
||||
- `product_title`: Full title of the product.
|
||||
- `ASIN`: The provided ASIN.
|
||||
- `product_url`: URL of the Amazon product page.
|
||||
- `brand`: Brand name.
|
||||
- `price_current_amount`: Current price.
|
||||
- `price_original_amount`: Original price (if applicable).
|
||||
- `price_discount_amount`: Discount amount (if applicable).
|
||||
- `rating_average`: Average star rating.
|
||||
- `rating_count`: Total number of ratings.
|
||||
- `featured`: Badges like "Amazon's Choice".
|
||||
- `color`: Color variant (if applicable).
|
||||
- `compatible_devices`: List of compatible devices (if applicable).
|
||||
- `product_description`: Full product description.
|
||||
- `special_features`: Highlighted features.
|
||||
- `style`: Style attribute (if applicable).
|
||||
- `material`: Material used (if applicable).
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
If an error occurs during script execution (e.g., network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check Output Content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. **Do not retry**; guide the user to re-check and provide the correct API Key.
|
||||
- If the output **does not contain** `"Invalid authorization"` but the task failed (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically try to re-execute the script once**.
|
||||
|
||||
2. **Retry Limit**:
|
||||
- Automatic retry is limited to **one time**. If the second attempt fails, stop retrying and report the specific error information to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Product Data Enrichment**: Retrieve full details for a list of ASINs to update an e-commerce database.
|
||||
2. **Price Comparison**: Lookup current Amazon prices for specific ASINs to compare with other retailers.
|
||||
3. **Review Monitoring**: Track changes in rating averages and review counts for key products.
|
||||
4. **Availability Checks**: Automatically verify if a specific product is currently in stock on Amazon.
|
||||
5. **Brand Analysis**: Identify the brand and manufacturer of products identified by ASIN.
|
||||
6. **Detailed Specifications**: Fetch material, style, and color information for catalog management.
|
||||
7. **Feature Highlighting**: Extract "special features" and detailed descriptions for marketing copy.
|
||||
8. **Compatibility Verification**: Check "compatible devices" for electronics or accessories.
|
||||
9. **Market Research**: Analyze featured badges like "Amazon's Choice" for specific product IDs.
|
||||
10. **URL Resolution**: Convert a list of ASINs into full Amazon product page URLs.
|
||||
@@ -0,0 +1,117 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams to handle multi-language content
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
# Amazon ASIN Lookup API Template ID
|
||||
TEMPLATE_ID = "77814333389670716"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
def run_amazon_asin_lookup(api_key, asin):
|
||||
"""
|
||||
Starts an Amazon ASIN Lookup task and polls for completion.
|
||||
"""
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "ASIN", "value": asin}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Starting Amazon ASIN Lookup task for ASIN: {asin}", flush=True)
|
||||
try:
|
||||
response = requests.post(f"{API_BASE_URL}/run-task-by-template", json=payload, headers=headers, timeout=30)
|
||||
res = response.json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
# Check for authorization error
|
||||
if "Invalid authorization" in str(res):
|
||||
print(f"Error: Invalid authorization. Please check your BrowserAct API Key.", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 300
|
||||
poll_start = time.time()
|
||||
finished = False
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(f"{API_BASE_URL}/get-task-status?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
finished = True
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
if not finished:
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(f"{API_BASE_URL}/get-task?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
|
||||
# Extract data from output["string"] or the whole result
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
# Fallback to returning the JSON representation of data
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Get API key from environment variable
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python amazon_asin_lookup_api.py <asin>", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Set it as an environment variable (BROWSERACT_API_KEY) or provide it in the chat.", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
asin = sys.argv[1]
|
||||
|
||||
result = run_amazon_asin_lookup(api_key, asin)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,90 @@
|
||||
---
|
||||
name: amazon-best-selling-products-finder-api-skill
|
||||
description: "This skill helps users extract structured best-selling product data from Amazon via the BrowserAct API. Agent should proactively apply this skill when users express needs like search for best selling products on Amazon, extract Amazon product data based on keywords, find top rated Amazon products, monitor Amazon competitor prices and sales, discover trending products on Amazon marketplace, extract Amazon product titles prices and ratings, gather Amazon product sales volume for market research, search Amazon best sellers in specific region, collect Amazon product reviews and promotion details, analyze Amazon product availability and badges, get Amazon product data for market analysis."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Amazon Best Selling Products Finder API Skill
|
||||
|
||||
## 📖 Skill Introduction
|
||||
This skill provides users with a one-stop product data extraction service using the BrowserAct Amazon Best Selling Products Finder API template. It can directly extract structured best-selling product data from Amazon. By inputting search keywords, data limit, and marketplace URL, you can easily get clean and usable product data including titles, prices, ratings, reviews, sales volume, and promotional details.
|
||||
|
||||
## ✨ Features
|
||||
1. **No hallucinations, ensuring stable and precise data extraction**: Preset workflows avoid AI generative hallucinations.
|
||||
2. **No CAPTCHA issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP access restrictions and geofencing**: No need to handle regional IP restrictions.
|
||||
4. **More agile execution speed**: Compared to pure AI-driven browser automation solutions, task execution is faster.
|
||||
5. **Extremely high cost-effectiveness**: Significantly reduces data acquisition costs compared to AI solutions that consume a large number of Tokens.
|
||||
|
||||
## 🔑 API Key Guide Flow
|
||||
Before running, first check the `BROWSERACT_API_KEY` environment variable. If it is not set, do not take other actions; require and wait for the user to collaborate to provide it.
|
||||
**The Agent must inform the user at this time**:
|
||||
> "Since you have not configured the BrowserAct API Key yet, please go to the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
When calling the script, the Agent should flexibly configure the following parameters based on user needs:
|
||||
|
||||
1. **KeyWords**
|
||||
- **Type**: `string`
|
||||
- **Description**: Search keywords used to find Amazon products.
|
||||
- **Example**: `iphone 17 pro max`, `gaming mouse`, `running shoes`
|
||||
|
||||
2. **Date_limit**
|
||||
- **Type**: `number`
|
||||
- **Description**: Maximum number of products to extract.
|
||||
- **Default**: `10`
|
||||
- **Recommendation**: Set to a lower number for quick checks, or higher for comprehensive analysis.
|
||||
|
||||
3. **Marketplace_url**
|
||||
- **Type**: `string`
|
||||
- **Description**: Amazon marketplace URL for region-specific searches.
|
||||
- **Default**: `https://www.amazon.com`
|
||||
- **Example**: `https://www.amazon.co.uk`, `https://www.amazon.de`
|
||||
|
||||
## 🚀 Call Method (Recommended)
|
||||
The Agent should execute the following standalone script to achieve "one command to get results":
|
||||
|
||||
```bash
|
||||
# Call example
|
||||
python -u ./scripts/amazon_best_selling_products_finder_api.py "search keywords" limit "marketplace_url"
|
||||
```
|
||||
|
||||
### ⏳ Running Status Monitoring
|
||||
Since this task involves automated browser operations, it may take a long time (several minutes). The script will **continuously output status logs with timestamps** (e.g. `[14:30:05] Task Status: running`) during execution.
|
||||
**Agent notice**:
|
||||
- While waiting for the script to return the result, please keep an eye on the terminal output.
|
||||
- As long as the terminal is still outputting new status logs, it means the task is running normally. Please do not mistakenly judge it as a deadlock or unresponsive.
|
||||
- If the status remains unchanged for a long time or the script stops outputting and no result is returned, the retry mechanism can be triggered.
|
||||
|
||||
## 📊 Data Output Description
|
||||
After successful execution, the script will parse and print the result directly from the API response. The result contains:
|
||||
- `title`: Product title
|
||||
- `brand`: Brand name
|
||||
- `list_price`: Original list price
|
||||
- `current_price`: Current selling price
|
||||
- `star_rating`: Average star rating
|
||||
- `review_count`: Total review count
|
||||
- `past_month_sales`: Sales volume in the past month
|
||||
- `availability`: Stock status
|
||||
- `promotion`: Promotional offers
|
||||
- `asin`: Amazon Standard Identification Number
|
||||
- `category`: Product category
|
||||
- `badge`: Badges like Amazon's Choice
|
||||
- `product_url`: Direct link to the product
|
||||
|
||||
## ⚠️ Error Handling & Retry Mechanism
|
||||
During the execution of the script, if an error occurs (such as network fluctuation or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check the output content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. At this time, **do not retry**, and guide the user to recheck and provide the correct API Key.
|
||||
- If the output **does not contain** `"Invalid authorization"` but the task execution fails (for example, the output starts with `Error:` or the returned result is empty), the Agent should **automatically try to execute the script again once**.
|
||||
|
||||
2. **Retry limit**:
|
||||
- Automatic retry is limited to **once**. If the second attempt still fails, stop retrying and report the specific error message to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Market Research**: Extract product listings and ratings to analyze the current market for specific keywords.
|
||||
2. **Competitor Analysis**: Monitor competitor pricing, discounts, and sales volume over time.
|
||||
3. **Trending Products Discovery**: Find the best-selling and highly rated products within a specific category.
|
||||
4. **Price Monitoring**: Track current prices and list prices to optimize purchasing strategies.
|
||||
5. **Cross-Region Analysis**: Compare product availability and pricing across different Amazon marketplaces.
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
TEMPLATE_ID = "85569073892601737"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
def run_amazon_best_selling_products_finder_task(api_key, keywords, limit=10, marketplace_url="https://www.amazon.com"):
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "KeyWords", "value": keywords},
|
||||
{"name": "Date_limit", "value": str(limit)},
|
||||
{"name": "Marketplace_url", "value": marketplace_url}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Start Task", flush=True)
|
||||
try:
|
||||
res = requests.post(f"{API_BASE_URL}/run-task-by-template", json=payload, headers=headers, timeout=30).json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 300
|
||||
poll_start = time.time()
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(f"{API_BASE_URL}/get-task-status?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(f"{API_BASE_URL}/get-task?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
|
||||
# Extract data from output["string"] as requested
|
||||
# Usually task_info['output']['string'] contains the result
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Prioritize command line API key, then environment variable
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python amazon_best_selling_products_finder_api.py <keywords> [limit] [marketplace_url]", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Provide it to me or set it as an environment variable (BROWSERACT_API_KEY).", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
keywords = sys.argv[1]
|
||||
limit = sys.argv[2] if len(sys.argv) > 2 else 10
|
||||
marketplace_url = sys.argv[3] if len(sys.argv) > 3 else "https://www.amazon.com"
|
||||
|
||||
result = run_amazon_best_selling_products_finder_task(api_key, keywords, limit, marketplace_url)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,87 @@
|
||||
---
|
||||
name: amazon-buy-box-monitor-api-skill
|
||||
description: "This skill helps users extract basic product details other sellers prices and seller ratings from Amazon via ASIN automatically using the BrowserAct API. Agent should proactively apply this skill when users express needs like query Amazon buy box information, monitor Amazon product prices, extract Amazon product details by ASIN, check other sellers prices on Amazon, get Amazon seller ratings and feedback count, monitor buy box ownership for a specific ASIN, track Amazon fulfillment methods for competitors, compare Amazon product prices across different sellers, retrieve Amazon buy box availability status, analyze Amazon seller profile details."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Amazon Buy Box Monitor API Skill
|
||||
|
||||
## 📖 Introduction
|
||||
This skill provides users with an automated Amazon Buy Box monitoring service using the BrowserAct API template. It can directly extract structured data including basic product details, other sellers' prices, and seller ratings from Amazon via a specific ASIN. No coding or proxies are required, and users only need to provide the ASIN and an optional marketplace URL to retrieve clean and usable data.
|
||||
|
||||
## ✨ Features
|
||||
1. **No hallucinations, ensuring stable and accurate data extraction**: Pre-set workflows avoid AI generative hallucinations.
|
||||
2. **No CAPTCHA issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP access restrictions or geo-fencing**: No need to handle regional IP restrictions.
|
||||
4. **More agile execution speed**: Faster task execution compared to purely AI-driven browser automation solutions.
|
||||
5. **Extremely high cost-effectiveness**: Significantly reduces data acquisition costs compared to AI solutions that consume massive amounts of tokens.
|
||||
|
||||
## 🔑 API Key Guidance
|
||||
Before running, you must check the `BROWSERACT_API_KEY` environment variable. If it is not set, do not take any other actions; you must prompt and wait for the user to provide it.
|
||||
**The Agent must inform the user at this time**:
|
||||
> "Since you have not configured the BrowserAct API Key, please go to the [BrowserAct Console](https://www.browseract.com/reception/integrations) first to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
When calling the script, the Agent should flexibly configure the following parameters based on the user's needs:
|
||||
|
||||
1. **ASIN**
|
||||
- **Type**: `string`
|
||||
- **Description**: The Amazon Standard Identification Number. This is the unique identifier for the product on Amazon.
|
||||
- **Example**: `B005O2ZU68`
|
||||
- **Required**: Yes
|
||||
|
||||
2. **Marketplace_url**
|
||||
- **Type**: `string`
|
||||
- **Description**: The Amazon marketplace URL indicating the region.
|
||||
- **Default value**: `https://amazon.com/`
|
||||
- **Example**: `https://amazon.co.uk/`
|
||||
- **Required**: Yes
|
||||
|
||||
## 🚀 Invocation Method (Recommended)
|
||||
The Agent should execute the following standalone script to achieve "one command to get results":
|
||||
|
||||
```bash
|
||||
# Example invocation
|
||||
python -u ./scripts/amazon_buy_box_monitor_api.py "ASIN" "Marketplace_url"
|
||||
```
|
||||
|
||||
### ⏳ Running Status Monitoring
|
||||
Since this task involves automated browser operations, it may take some time (several minutes). The script will **continuously output status logs with timestamps** while running (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent Instructions**:
|
||||
- While waiting for the script to return results, please keep paying attention to the terminal output.
|
||||
- As long as the terminal is still outputting new status logs, it means the task is running normally. Do not misjudge it as deadlocked or unresponsive.
|
||||
- If the status remains unchanged for a long time or the script stops outputting without returning a result, then consider triggering the retry mechanism.
|
||||
|
||||
## 📊 Data Output Description
|
||||
After successful execution, the script will parse and print the result directly from the API response. The result includes:
|
||||
- `asin`: The Amazon Standard Identification Number
|
||||
- `product_title`: The title of the product
|
||||
- `buy_box_owner`: The owner of the buy box
|
||||
- `buy_box_price`: The current buy box price
|
||||
- `currency`: The currency of the price
|
||||
- `fulfillment_method`: The fulfillment method (e.g., FBA, FBM)
|
||||
- `availability_status`: Stock availability status
|
||||
- `other_sellers`: An array of other sellers including their name, price, shipping fee, and seller rating
|
||||
- `seller_info`: Detailed information about the main seller including rating and feedback count
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
If an error is encountered during the execution of the script (such as network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check output content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. In this case, **do not retry**, but guide the user to re-check and provide the correct API Key.
|
||||
- If the output **does not contain** `"Invalid authorization"` but the task fails (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically attempt to execute the script once more**.
|
||||
|
||||
2. **Retry limits**:
|
||||
- Automatic retry is limited to **only once**. If the second attempt still fails, stop retrying and report the specific error message to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Query Amazon buy box information**: Find out who currently owns the buy box for a specific ASIN.
|
||||
2. **Monitor Amazon product prices**: Track the current price and buy box price changes.
|
||||
3. **Extract Amazon product details by ASIN**: Get basic product information like title and brand.
|
||||
4. **Check other sellers prices on Amazon**: Analyze pricing strategies of competitors for the same product.
|
||||
5. **Get Amazon seller ratings and feedback count**: Evaluate the reputation of sellers on the listing.
|
||||
6. **Monitor buy box ownership for a specific ASIN**: Check if a particular seller maintains the buy box.
|
||||
7. **Track Amazon fulfillment methods for competitors**: Determine whether competitors are using FBA or FBM.
|
||||
8. **Compare Amazon product prices across different sellers**: View shipping fees and total prices from multiple sellers.
|
||||
9. **Retrieve Amazon buy box availability status**: Check if the product is in stock or backordered.
|
||||
10. **Analyze Amazon seller profile details**: Extract detailed seller info and recent feedback summaries.
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
TEMPLATE_ID = "87215742629531801"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
def run_amazon_buy_box_monitor_task(api_key, asin, marketplace_url="https://amazon.com/"):
|
||||
"""
|
||||
Starts an Amazon Buy Box Monitor task and polls for completion.
|
||||
"""
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "ASIN", "value": asin},
|
||||
{"name": "Marketplace_url", "value": marketplace_url}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Starting Amazon Buy Box Monitor task for ASIN: {asin}", flush=True)
|
||||
try:
|
||||
res = requests.post(f"{API_BASE_URL}/run-task-by-template", json=payload, headers=headers, timeout=30).json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
if "Invalid authorization" in str(res):
|
||||
print(f"Error: Invalid authorization. Please check your BrowserAct API Key.", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 300
|
||||
poll_start = time.time()
|
||||
finished = False
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(f"{API_BASE_URL}/get-task-status?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
finished = True
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
if not finished:
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(f"{API_BASE_URL}/get-task?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python amazon_buy_box_monitor_api.py <asin> [marketplace_url]", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Provide it to me or set it as an environment variable (BROWSERACT_API_KEY).", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
asin = sys.argv[1]
|
||||
marketplace_url = sys.argv[2] if len(sys.argv) > 2 else "https://amazon.com/"
|
||||
|
||||
result = run_amazon_buy_box_monitor_task(api_key, asin, marketplace_url)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,5 @@
|
||||
# Template for BrowserAct API Key Configuration
|
||||
# Copy this file to .env and replace the placeholder with your actual API key
|
||||
# Get your API key from: https://www.browseract.com/reception/integrations
|
||||
|
||||
BROWSERACT_API_KEY=YOUR_API_KEY_HERE
|
||||
@@ -0,0 +1,127 @@
|
||||
---
|
||||
name: amazon-competitor-analyzer
|
||||
description: Scrapes Amazon product data from ASINs using browseract.com automation API and performs surgical competitive analysis. Compares specifications, pricing, review quality, and visual strategies to identify competitor moats and vulnerabilities.
|
||||
metadata: {"openclaw":{"emoji":"📊","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Amazon Competitor Analyzer
|
||||
|
||||
This skill scrapes Amazon product data from user-provided ASINs using browseract.com's browser automation API and performs deep competitive analysis.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
- Competitive research: Input multiple ASINs to understand market landscape
|
||||
- Pricing strategy analysis: Compare price bands across similar products
|
||||
- Specification benchmarking: Deep dive into technical specs and feature differences
|
||||
- Review insights: Analyze review quality, quantity, and sentiment patterns
|
||||
- Market opportunity discovery: Identify gaps and potential threats
|
||||
|
||||
## What This Skill Does
|
||||
|
||||
1. **ASIN Data Collection**: Extract product title, price, rating, review count, images
|
||||
2. **Specification Extraction**: Deep extraction of technical specs, features, and materials
|
||||
3. **Review Quality Analysis**: Analyze review patterns, keywords, and sentiment
|
||||
4. **Multi-Dimensional Comparison**: Side-by-side comparison of key metrics
|
||||
5. **Moat Identification**: Identify core competitive advantages and barriers
|
||||
6. **Vulnerability Discovery**: Find competitor weaknesses and market opportunities
|
||||
|
||||
## Features
|
||||
|
||||
1. **Stable and accurate data extraction**: Pre-set workflows ensure consistent results.
|
||||
2. **Browser automation**: Uses BrowserAct's automated browser instances for reliable data collection.
|
||||
3. **Global accessibility**: BrowserAct provides servers in multiple regions.
|
||||
4. **Fast execution**: Optimized workflow templates complete tasks quickly.
|
||||
5. **Cost efficient**: Reduces manual research time and associated costs.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### BrowserAct.com Account Setup
|
||||
|
||||
You need a BrowserAct.com account and API key:
|
||||
|
||||
1. Visit [browseract.com](https://browseract.com)
|
||||
2. Sign up for an account
|
||||
3. Navigate to [API Settings](https://www.browseract.com/reception/integrations)
|
||||
4. Generate an API key
|
||||
|
||||
### Environment Configuration
|
||||
|
||||
Copy the `.env.example` file to `.env` and add your API key:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# Edit .env and replace YOUR_API_KEY_HERE with your actual API key
|
||||
```
|
||||
|
||||
Or set as environment variable:
|
||||
|
||||
```bash
|
||||
export BROWSERACT_API_KEY="your-api-key-here"
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic Analysis
|
||||
|
||||
```bash
|
||||
python amazon-competitor-analyzer/amazon_competitor_analyzer.py B09G9GB4MG
|
||||
```
|
||||
|
||||
### Multiple Products
|
||||
|
||||
```bash
|
||||
python amazon-competitor-analyzer/amazon_competitor_analyzer.py B09G9GB4MG B07ABC11111 B08N5WRWNW
|
||||
```
|
||||
|
||||
### With Output Directory
|
||||
|
||||
```bash
|
||||
python amazon-competitor-analyzer/amazon_competitor_analyzer.py B09G9GB4MG -o ./output
|
||||
```
|
||||
|
||||
### Output Formats
|
||||
|
||||
- **CSV**: Structured data table
|
||||
- **Markdown**: Comprehensive report
|
||||
- **JSON**: Raw data with analysis
|
||||
|
||||
## Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| asins | string | - | One or more Amazon ASINs to analyze |
|
||||
| --output, -o | string | ./output | Output directory |
|
||||
| --format | string | all | Output format (csv/markdown/json/all) |
|
||||
| --api-key | string | env | BrowserAct API key |
|
||||
|
||||
## Dependencies
|
||||
|
||||
This skill requires the following Python packages:
|
||||
|
||||
```bash
|
||||
pip install requests
|
||||
```
|
||||
|
||||
Optional (for automatic .env loading):
|
||||
```bash
|
||||
pip install python-dotenv
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Required | Description |
|
||||
|----------|----------|-------------|
|
||||
| BROWSERACT_API_KEY | Yes | Your BrowserAct API key. Get it from [BrowserAct Console](https://www.browseract.com/reception/integrations) |
|
||||
|
||||
## Error Handling
|
||||
|
||||
- **Invalid API Key**: Check BROWSERACT_API_KEY environment variable
|
||||
- **Network Error**: Verify internet connection
|
||||
- **Rate Limit**: Wait and retry with exponential backoff
|
||||
- **Invalid ASIN**: Verify ASIN format (10 alphanumeric characters)
|
||||
|
||||
---
|
||||
|
||||
**Version**: 1.0.0
|
||||
**Updated**: 2026-02-09
|
||||
**Template ID**: `77814333389670716`
|
||||
@@ -0,0 +1,367 @@
|
||||
"""
|
||||
Amazon Competitor Analyzer Skill
|
||||
===============================
|
||||
Scrapes Amazon product data from ASINs using BrowserAct API
|
||||
and performs surgical competitive analysis.
|
||||
|
||||
Author: OpenCode
|
||||
Version: 1.0.0
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import csv
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Any
|
||||
from pathlib import Path
|
||||
|
||||
# Try to load .env file if python-dotenv is available
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
# Load .env from same directory as script
|
||||
script_dir = Path(__file__).parent
|
||||
env_file = script_dir / ".env"
|
||||
if env_file.exists():
|
||||
load_dotenv(env_file)
|
||||
except ImportError:
|
||||
# If python-dotenv not available, check .env manually
|
||||
script_dir = Path(__file__).parent
|
||||
env_file = script_dir / ".env"
|
||||
if env_file.exists():
|
||||
with open(env_file) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#') and '=' in line:
|
||||
key, value = line.split('=', 1)
|
||||
os.environ.setdefault(key, value.strip())
|
||||
|
||||
|
||||
# Configuration
|
||||
BROWSERACT_API_KEY = os.getenv("BROWSERACT_API_KEY", "")
|
||||
WORKFLOW_TEMPLATE_ID = "77814333389670716"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
|
||||
class AmazonCompetitorAnalyzer:
|
||||
"""Main class for Amazon competitive analysis"""
|
||||
|
||||
def __init__(self, api_key: str = None, workflow_template_id: str = None):
|
||||
"""Initialize the analyzer with API credentials"""
|
||||
self.api_key = api_key or BROWSERACT_API_KEY
|
||||
self.workflow_template_id = workflow_template_id or WORKFLOW_TEMPLATE_ID
|
||||
self.headers = {
|
||||
"Authorization": f"Bearer {self.api_key}"
|
||||
}
|
||||
|
||||
def validate_asin(self, asin: str) -> bool:
|
||||
"""Validate ASIN format"""
|
||||
return len(asin) == 10 and asin.isalnum()
|
||||
|
||||
def extract_asins_from_text(self, text: str) -> List[str]:
|
||||
"""Extract ASINs from user input text"""
|
||||
import re
|
||||
# Match 10-character alphanumeric strings starting with B0 or similar
|
||||
asin_pattern = r'\b[B0][A-Z0-9]{9}\b'
|
||||
asins = re.findall(asin_pattern, text.upper())
|
||||
return list(set(asins)) # Remove duplicates
|
||||
|
||||
def submit_task(self, asin: str) -> Optional[str]:
|
||||
"""Submit scraping task for a single ASIN"""
|
||||
if not self.validate_asin(asin):
|
||||
print(f"Invalid ASIN: {asin}")
|
||||
return None
|
||||
|
||||
data = {
|
||||
"workflow_template_id": self.workflow_template_id,
|
||||
"input_parameters": [{"name": "ASIN", "value": asin}]
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{API_BASE_URL}/run-task-by-template",
|
||||
json=data,
|
||||
headers=self.headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
task_id = result.get("id")
|
||||
return task_id
|
||||
else:
|
||||
print(f"Failed to submit task for {asin}: {response.json().get('msg', 'Unknown error')}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error submitting task for {asin}: {e}")
|
||||
return None
|
||||
|
||||
def wait_for_task(self, task_id: str, timeout: int = 300) -> bool:
|
||||
"""Wait for task completion"""
|
||||
start_time = time.time()
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{API_BASE_URL}/get-task-status?task_id={task_id}",
|
||||
headers=self.headers,
|
||||
timeout=10
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
status = response.json().get("status")
|
||||
|
||||
if status == "finished":
|
||||
return True
|
||||
elif status in ["failed", "canceled"]:
|
||||
return False
|
||||
|
||||
time.sleep(3)
|
||||
|
||||
except Exception:
|
||||
time.sleep(5)
|
||||
|
||||
return False
|
||||
|
||||
def get_results(self, task_id: str) -> Optional[Dict]:
|
||||
"""Get task results"""
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{API_BASE_URL}/get-task?task_id={task_id}",
|
||||
headers=self.headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
except Exception as e:
|
||||
print(f"Error getting results: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def scrape_product(self, asin: str, wait_timeout: int = 300) -> Optional[Dict]:
|
||||
"""Scrape a single product"""
|
||||
# Submit task
|
||||
task_id = self.submit_task(asin)
|
||||
if not task_id:
|
||||
return None
|
||||
|
||||
# Wait for completion
|
||||
if not self.wait_for_task(task_id, wait_timeout):
|
||||
return None
|
||||
|
||||
# Get results
|
||||
results = self.get_results(task_id)
|
||||
return results
|
||||
|
||||
def scrape_multiple_products(self, asins: List[str], delay: int = 5) -> Dict[str, Any]:
|
||||
"""Scrape multiple products"""
|
||||
results = {}
|
||||
|
||||
for asin in asins:
|
||||
print(f"Processing: {asin}")
|
||||
|
||||
data = self.scrape_product(asin)
|
||||
results[asin] = data
|
||||
|
||||
if delay > 0 and asin != asins[-1]:
|
||||
time.sleep(delay)
|
||||
|
||||
return results
|
||||
|
||||
def analyze_competitive_position(self, products: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Analyze competitive positioning"""
|
||||
analysis = {
|
||||
"price_analysis": {},
|
||||
"rating_analysis": {},
|
||||
"market_leaders": {},
|
||||
"opportunities": []
|
||||
}
|
||||
|
||||
prices = []
|
||||
ratings = []
|
||||
|
||||
for asin, data in products.items():
|
||||
if data:
|
||||
try:
|
||||
product = data.get('results', {}).get('products', [{}])[0]
|
||||
price = product.get('pricing', {}).get('current_price', 0)
|
||||
rating = product.get('reviews', {}).get('average_rating', 0)
|
||||
reviews = product.get('reviews', {}).get('total_count', 0)
|
||||
brand = product.get('product_info', {}).get('brand', asin)
|
||||
|
||||
prices.append((asin, price, brand))
|
||||
ratings.append((asin, rating, reviews, brand))
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Sort by price
|
||||
prices.sort(key=lambda x: x[1])
|
||||
if prices:
|
||||
analysis["price_analysis"]["lowest"] = prices[0]
|
||||
analysis["price_analysis"]["highest"] = prices[-1]
|
||||
analysis["price_analysis"]["range"] = prices[-1][1] - prices[0][1]
|
||||
|
||||
# Sort by rating
|
||||
ratings.sort(key=lambda x: x[1], reverse=True)
|
||||
if ratings:
|
||||
analysis["rating_analysis"]["top_rated"] = ratings[0]
|
||||
analysis["rating_analysis"]["by_volume"] = sorted(ratings, key=lambda x: x[2], reverse=True)
|
||||
|
||||
return analysis
|
||||
|
||||
def generate_csv_report(self, products: Dict[str, Any], output_path: str):
|
||||
"""Generate CSV report"""
|
||||
with open(output_path, 'w', newline='', encoding='utf-8') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow([
|
||||
'ASIN', 'Product Title', 'Brand', 'Price ($)', 'Original Price ($)',
|
||||
'Discount (%)', 'Rating', 'Reviews Count', 'Weight', 'Features'
|
||||
])
|
||||
|
||||
for asin, data in products.items():
|
||||
if data:
|
||||
try:
|
||||
product = data.get('results', {}).get('products', [{}])[0]
|
||||
writer.writerow([
|
||||
asin,
|
||||
product.get('product_info', {}).get('title', 'N/A')[:100],
|
||||
product.get('product_info', {}).get('brand', 'N/A'),
|
||||
product.get('pricing', {}).get('current_price', 'N/A'),
|
||||
product.get('pricing', {}).get('original_price', 'N/A'),
|
||||
product.get('pricing', {}).get('discount_percent', 'N/A'),
|
||||
product.get('reviews', {}).get('average_rating', 'N/A'),
|
||||
product.get('reviews', {}).get('total_count', 'N/A'),
|
||||
product.get('specifications', {}).get('weight', 'N/A'),
|
||||
', '.join(product.get('specifications', {}).get('features', [])[:5])
|
||||
])
|
||||
except Exception:
|
||||
writer.writerow([asin, 'Error', '', '', '', '', '', '', '', ''])
|
||||
else:
|
||||
writer.writerow([asin, 'Failed', '', '', '', '', '', '', '', ''])
|
||||
|
||||
print(f"CSV report saved: {output_path}")
|
||||
|
||||
def generate_markdown_report(self, products: Dict[str, Any], output_path: str):
|
||||
"""Generate comprehensive markdown report"""
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write("# Amazon Competitive Analysis Report\n\n")
|
||||
f.write(f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
|
||||
f.write(f"**Products Analyzed:** {len(products)}\n\n")
|
||||
|
||||
# Summary table
|
||||
f.write("## Data Summary\n\n")
|
||||
f.write("| ASIN | Brand | Price | Rating | Reviews |\n")
|
||||
f.write("|------|-------|-------|--------|---------|\n")
|
||||
|
||||
for asin, data in products.items():
|
||||
if data:
|
||||
try:
|
||||
product = data.get('results', {}).get('products', [{}])[0]
|
||||
info = product.get('product_info', {})
|
||||
pricing = product.get('pricing', {})
|
||||
reviews = product.get('reviews', {})
|
||||
|
||||
f.write(f"| {asin} | {info.get('brand', 'N/A')} | "
|
||||
f"${pricing.get('current_price', 'N/A')} | "
|
||||
f"{reviews.get('average_rating', 'N/A')}/5 | "
|
||||
f"{reviews.get('total_count', 'N/A'):,} |\n")
|
||||
except Exception:
|
||||
f.write(f"| {asin} | Error | - | - | - |\n")
|
||||
else:
|
||||
f.write(f"| {asin} | Failed | - | - | - |\n")
|
||||
|
||||
# Detailed analysis
|
||||
f.write("\n## Detailed Analysis\n\n")
|
||||
|
||||
competitive_analysis = self.analyze_competitive_position(products)
|
||||
|
||||
f.write("### Price Positioning\n")
|
||||
if competitive_analysis.get("price_analysis"):
|
||||
pa = competitive_analysis["price_analysis"]
|
||||
if "lowest" in pa:
|
||||
f.write(f"- Lowest Price: {pa['lowest'][2]} at ${pa['lowest'][1]}\n")
|
||||
if "highest" in pa:
|
||||
f.write(f"- Highest Price: {pa['highest'][2]} at ${pa['highest'][1]}\n")
|
||||
|
||||
f.write("\n### Rating Leaders\n")
|
||||
if competitive_analysis.get("rating_analysis"):
|
||||
ra = competitive_analysis["rating_analysis"]
|
||||
if "top_rated" in ra:
|
||||
f.write(f"- Highest Rated: {ra['top_rated'][3]} at {ra['top_rated'][1]}/5\n")
|
||||
|
||||
f.write("\n---\n")
|
||||
f.write(f"*Generated by Amazon Competitor Analyzer*\n")
|
||||
|
||||
print(f"Markdown report saved: {output_path}")
|
||||
|
||||
def generate_json_report(self, products: Dict[str, Any], output_path: str):
|
||||
"""Generate JSON report"""
|
||||
report_data = {
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
"products_analyzed": len(products),
|
||||
"products": products,
|
||||
"analysis": self.analyze_competitive_position(products)
|
||||
}
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(report_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"JSON report saved: {output_path}")
|
||||
|
||||
|
||||
def analyze_asins(asins: List[str], output_dir: str = None) -> Dict[str, Any]:
|
||||
"""Main function to analyze multiple ASINs"""
|
||||
analyzer = AmazonCompetitorAnalyzer()
|
||||
|
||||
print(f"Analyzing {len(asins)} ASINs...")
|
||||
|
||||
# Scrape products
|
||||
products = analyzer.scrape_multiple_products(asins)
|
||||
|
||||
# Generate reports if output directory specified
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
base_path = os.path.join(output_dir, "amazon_analysis")
|
||||
analyzer.generate_csv_report(products, f"{base_path}.csv")
|
||||
analyzer.generate_markdown_report(products, f"{base_path}.md")
|
||||
analyzer.generate_json_report(products, f"{base_path}.json")
|
||||
|
||||
return products
|
||||
|
||||
|
||||
def main():
|
||||
"""CLI entry point"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Amazon Competitor Analyzer")
|
||||
parser.add_argument("asins", nargs="+", help="ASINs to analyze")
|
||||
parser.add_argument("-o", "--output", default=".", help="Output directory")
|
||||
parser.add_argument("-k", "--api-key", help="BrowserAct API key")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Run analysis
|
||||
api_key = args.api_key or BROWSERACT_API_KEY
|
||||
analyzer = AmazonCompetitorAnalyzer(api_key=api_key)
|
||||
|
||||
print(f"Analyzing {len(args.asins)} ASINs...")
|
||||
products = analyzer.scrape_multiple_products(args.asins)
|
||||
|
||||
if args.output:
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
base_path = os.path.join(args.output, "amazon_analysis")
|
||||
analyzer.generate_csv_report(products, f"{base_path}.csv")
|
||||
analyzer.generate_markdown_report(products, f"{base_path}.md")
|
||||
analyzer.generate_json_report(products, f"{base_path}.json")
|
||||
|
||||
print(f"\nAnalysis complete! Analyzed {len(args.asins)} products.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,201 @@
|
||||
---
|
||||
name: amazon-listing-competitor-analysis-skill
|
||||
description: "This skill helps users analyze Amazon competitor listings by ASIN and produce structured competitive intelligence plus strategic opportunity points for their own go-to-market. The Agent should proactively apply this skill when users want to analyze a competitor Amazon listing by ASIN, understand what a top-ranked product does right in content keywords or visuals, find market gaps and unmet buyer needs, turn competitor research into opportunity maps for their brand, identify keyword placement patterns on rival listings, extract SEO insights from Amazon product pages, reverse-engineer competitor bullet and title strategies, mine competitor reviews for buyer psychology, compare seller and A plus content patterns, run gap analysis before launching a new SKU, research why a listing wins conversion signals, synthesize whitespace you can own versus the diagnosed listing, or say just look at this ASIN with a competitive or optimization angle."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Amazon Listing Competitor Analysis
|
||||
|
||||
## 📖 Brief
|
||||
This skill runs a two-phase workflow on a single **competitor** Amazon listing. **Phase 1** uses the BrowserAct Amazon Listing Extractor for SEO template to pull visible product data from that listing. **Phase 2** diagnoses what that competitor does well and where the market shows gaps, then closes with **your strategic opportunity points** (how you can win next to them). Do **not** end with instructions that read like editing or rewriting **this competitor's** listing; the analyzed ASIN is evidence only. Final narrative output should be grounded in extracted data, not generic claims.
|
||||
|
||||
## ✨ Features
|
||||
1. **No hallucinations, ensuring stable and accurate data extraction**: Pre-set workflows avoid AI generative hallucinations.
|
||||
2. **No CAPTCHA issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP restrictions or geo-blocking**: No need to deal with regional IP restrictions or geofencing.
|
||||
4. **Faster execution**: Tasks execute faster compared to purely AI-driven browser automation solutions.
|
||||
5. **Extremely high cost-efficiency**: Significantly reduces data acquisition costs compared to AI solutions that consume massive amounts of tokens.
|
||||
|
||||
## 🔑 API Key Guide
|
||||
Before running, you must check the `BROWSERACT_API_KEY` environment variable. If it is not set, do not take other actions first; you should ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key yet, please go to the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
When calling the script, the Agent should flexibly configure the following parameters based on user needs:
|
||||
|
||||
1. **ASIN**
|
||||
- **Type**: `string`
|
||||
- **Description**: The ASIN (Amazon Standard Identification Number) of the Amazon product to analyze.
|
||||
- **Example**: `B0CS62LY6P`
|
||||
- **Required**: Yes
|
||||
|
||||
2. **Marketplace_url**
|
||||
- **Type**: `string`
|
||||
- **Description**: The base URL of the Amazon marketplace. Use the correct regional site for the listing.
|
||||
- **Example**: `https://www.amazon.com/`, `https://www.amazon.de/`
|
||||
- **Default**: `https://www.amazon.com/`
|
||||
|
||||
## 🚀 Invocation Method
|
||||
Run Phase 1 extraction with the script below. After structured data is returned, the Agent performs Phase 2 analysis using the framework in **Competitive Analysis Framework (Phase 2)**. The closing section must synthesize **opportunity points for the user's business**, not a checklist of edits applied to the competitor page under review.
|
||||
|
||||
```bash
|
||||
python -u ./scripts/amazon_listing_competitor_analysis.py "B0CS62LY6P" "https://www.amazon.com/"
|
||||
```
|
||||
|
||||
When only the ASIN is needed, the marketplace argument may be omitted; the script defaults to `https://www.amazon.com/`.
|
||||
|
||||
### ⏳ Running Status Monitoring
|
||||
Since this task involves automated browser operations, it may take a long time (several minutes). The script will **continuously output status logs with timestamps** while running (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent guidelines**:
|
||||
- While waiting for the script to return results, please keep an eye on the terminal output.
|
||||
- As long as the terminal continues to output new status logs, it means the task is running normally. Do not misjudge it as a deadlock or unresponsiveness.
|
||||
- If the status remains unchanged for a long time or the script stops outputting without returning a result, only then consider triggering the retry mechanism.
|
||||
|
||||
## 📊 Data Output
|
||||
Upon successful execution, the script prints the API result string (or full task JSON if no string field is present). Typical fields include:
|
||||
- `asin`, `title`, `product_url`, `brand`, `price`, `coupon_text`, `rating`, `review_count`, `best_sellers_rank`, `availability`, `prime_eligible`
|
||||
- `description`, `short_description`, `category`, `key_features`, `bullet_points`
|
||||
- `main_image_url`, `additional_image_urls`, `seller_name`, `ships_from`, `sold_by`
|
||||
- `specifications`, `product_details`, `attributes`, and review-related blocks (reviewer, content, date, helpful votes, etc.)
|
||||
|
||||
Use this payload as the single source of truth for Phase 2; do not invent listing facts.
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
During script execution, if errors occur (such as network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check the output content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. At this point, **do not retry**, but guide the user to recheck and provide the correct API Key.
|
||||
- If the output **contains** `"concurrent"` or `"too many running tasks"` or similar concurrency limit messages, it means the concurrent task limit for the current subscription plan has been reached. **Do not retry**; guide the user to upgrade their plan.
|
||||
**Agent must inform the user**:
|
||||
> "The current task cannot be executed because your BrowserAct account has reached the limit of concurrent tasks. Please go to the [BrowserAct Plan Upgrade Page](https://www.browseract.com/reception/recharge) to upgrade your subscription plan and enjoy more concurrent task benefits."
|
||||
- If the output **does not contain** the above error keywords but the task fails (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically try to run the script once more**.
|
||||
|
||||
2. **Retry limit**:
|
||||
- Automatic retry is limited to **once**. If the second attempt still fails, stop retrying and report the specific error message to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Competitor listing teardown**: Analyze one ASIN to see title formula, bullets, and differentiation language.
|
||||
2. **Keyword placement audit**: Map where primary and long-tail terms appear across title, bullets, and description or A+ content.
|
||||
3. **Visual strategy review**: Infer image narrative, infographic highlights, and video approach from extracted media data.
|
||||
4. **Buyer-validated selling points**: Use high-helpful positive reviews to confirm what buyers value versus what the listing emphasizes.
|
||||
5. **Unmet needs mining**: Use three-star and mixed reviews to find feature and expectation gaps.
|
||||
6. **Pre-launch gap analysis**: Compare a planned positioning against a top competitor's listing structure.
|
||||
7. **Cross-marketplace research**: Run the same ASIN on different regional Amazon URLs for localized copy signals.
|
||||
8. **Opportunity backlog from a rival listing**: Turn extracted facts and gaps into a prioritized map of positioning, search, creative, and offer opportunities for your side of the market.
|
||||
9. **SEO and conversion benchmarking**: Relate BSR, rating volume, and copy patterns without guessing unavailable metrics.
|
||||
10. **Review-driven objection handling**: Surface recurring complaints to address in copy or images.
|
||||
|
||||
## 🧠 Competitive Analysis Framework (Phase 2)
|
||||
After extraction succeeds, work through each dimension below. Every insight must be grounded in the actual extracted data.
|
||||
|
||||
### Layer 1 — What the Competitor Did Right
|
||||
|
||||
**1. Content Strategy**
|
||||
|
||||
- **Title formula**: Information order, primary keyword placement, brand-first vs feature-first vs use-case-first.
|
||||
- **Bullet priority**: What Bullet 1 leads with; selling point order across bullets (signal of tested conversion order).
|
||||
- **Differentiation language**: How generic category features are phrased to sound distinct.
|
||||
- **A+ content**: Modules implied by extracted content (comparison table, brand story, lifestyle, spec callouts).
|
||||
|
||||
**2. Keyword Placement Strategy**
|
||||
|
||||
Map *where* terms appear (not only which terms exist):
|
||||
|
||||
- Title (first 80 chars) → primary ranking bets
|
||||
- Bullets 1–2 → secondary high-weight terms
|
||||
- Bullets 3–5 → long-tail and use-case terms
|
||||
- Description / A+ → supplementary terms and synonyms
|
||||
|
||||
**3. Visual Content Strategy**
|
||||
|
||||
- **Image narrative arc**: Sequence story (hero, lifestyle, pain point, specs, size comparison, social proof, guarantee).
|
||||
- **Infographic data**: Numbers or attributes highlighted and how they are presented.
|
||||
- **Video** (if present in data): Hook length, demo vs lifestyle, subtitles.
|
||||
- **Overall style**: Premium, approachable, technical, lifestyle-focused.
|
||||
|
||||
**4. Buyer-Validated Selling Points**
|
||||
|
||||
From four- to five-star reviews with high helpful votes:
|
||||
|
||||
- What reviewers praise that the listing underplays
|
||||
- Unexpected benefits buyers mention
|
||||
|
||||
### Layer 2 — What the Market Lacks
|
||||
|
||||
**5. Unmet Buyer Needs**
|
||||
|
||||
From three-star reviews and recurring themes in low stars (non-defect noise):
|
||||
|
||||
- "I wish it had…", "Would be five stars if…", "Good but not great because…"
|
||||
|
||||
**6. Keyword Gaps**
|
||||
|
||||
- Natural search terms buyers would use that the listing does not cover
|
||||
- High-traffic angles the data suggests but copy does not foreground
|
||||
|
||||
**7. Visual Content Gaps**
|
||||
|
||||
- Weak or missing context in existing images
|
||||
- Absent image types (use-case, comparison, real-world scale)
|
||||
|
||||
### Required Output Format (Phase 2)
|
||||
Produce the analysis using this structure. Be specific and quote or paraphrase extracted fields and reviews where useful. The final block is **your opportunity synthesis**; avoid imperatives that sound like "change this competitor's bullet five" or any direct edit list for the ASIN being studied.
|
||||
|
||||
```
|
||||
Competitor ASIN: [ASIN] | Brand: [brand] | BSR: [rank] | Rating: [x.x] ([N] reviews)
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
✅ WHAT THIS COMPETITOR DOES RIGHT
|
||||
|
||||
Content Strategy:
|
||||
- Title formula: [describe the pattern and keyword placement]
|
||||
- Bullet priority: [what each bullet leads with and the logic behind the order]
|
||||
- Standout phrasing: [specific language worth noting or borrowing]
|
||||
- A+ modules: [which are used and what they emphasize]
|
||||
|
||||
Keyword Placement:
|
||||
- Primary (title, first 80 chars): [keywords]
|
||||
- High-weight (Bullets 1–2): [terms]
|
||||
- Long-tail (Bullets 3–5): [terms]
|
||||
- Supplementary (description/A+): [terms]
|
||||
|
||||
Visual Strategy:
|
||||
- Image sequence: [describe the narrative arc across images]
|
||||
- Infographic highlights: [what data/specs are called out]
|
||||
- Video: [approach if present, or "none"]
|
||||
|
||||
Buyer-Validated Selling Points:
|
||||
- "[specific insight from high-helpful reviews]"
|
||||
- "[another insight]"
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
🕳️ MARKET GAPS (OBSERVED ON THIS COMPETITOR LISTING)
|
||||
|
||||
Content gap: [selling points or use cases their copy under-serves, as seen in extracted text]
|
||||
Keyword gap: [search intents or terms weakly covered on their page — note buyer language from reviews where possible]
|
||||
Visual gap: [image or video proof types missing or weak on their gallery or A+]
|
||||
Unmet buyer needs: [recurring themes from 3-star and mixed reviews, quoted or paraphrased]
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
🎯 YOUR STRATEGIC OPPORTUNITY POINTS (FOR YOUR BRAND OR ROADMAP — NOT EDITS TO THIS LISTING)
|
||||
|
||||
The ASIN above is the competitor under diagnosis. Below, translate gaps into **where you can win**; do not phrase outcomes as rewriting their bullets or their title.
|
||||
|
||||
Positioning and messaging whitespace:
|
||||
- [Claim, use case, or audience angle they under-own; why it is an opening for you]
|
||||
|
||||
Search and intent capture:
|
||||
- [Queries or intents implied by reviews or category that their listing weakly serves; how you could own a different slice of demand]
|
||||
|
||||
Trust, proof, and creative differentiation:
|
||||
- [Proof points, demos, or gallery angles they lack that you could credibly own]
|
||||
|
||||
Product, offer, or bundle opportunity:
|
||||
- [Unmet needs from reviews that map to a SKU, variant, bundle, warranty, or service on your side — stay factual to extracted complaints and wishes]
|
||||
|
||||
Competitive strengths to respect or neutralize:
|
||||
- [What this competitor does so well in copy, visuals, or social proof that you should assume as the bar before claiming superiority]
|
||||
```
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
TEMPLATE_ID = "88457577087632388"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
|
||||
def run_amazon_listing_competitor_analysis_task(api_key, asin, marketplace_url="https://www.amazon.com/"):
|
||||
"""
|
||||
Starts an Amazon listing extraction task for competitor analysis and polls for completion.
|
||||
Returns structured data as a string, or None on failure.
|
||||
"""
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "ASIN", "value": asin},
|
||||
{"name": "Marketplace_url", "value": marketplace_url}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Start Task", flush=True)
|
||||
try:
|
||||
res = requests.post(
|
||||
f"{API_BASE_URL}/run-task-by-template",
|
||||
json=payload, headers=headers, timeout=30
|
||||
).json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
res_str = str(res)
|
||||
if "Invalid authorization" in res_str:
|
||||
print("Error: Invalid authorization. Please check your BrowserAct API Key.", flush=True)
|
||||
elif "concurrent" in res_str.lower() or "too many running tasks" in res_str.lower():
|
||||
print("Error: Concurrent task limit reached. Please upgrade your plan at https://www.browseract.com/reception/recharge", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 900
|
||||
poll_start = time.time()
|
||||
finished = False
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(
|
||||
f"{API_BASE_URL}/get-task-status?task_id={task_id}",
|
||||
headers=headers, timeout=30
|
||||
).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
finished = True
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
if not finished:
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(
|
||||
f"{API_BASE_URL}/get-task?task_id={task_id}",
|
||||
headers=headers, timeout=30
|
||||
).json()
|
||||
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python amazon_listing_competitor_analysis.py <asin> [marketplace_url]", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Provide it to me or set it as an environment variable (BROWSERACT_API_KEY).", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
asin = sys.argv[1]
|
||||
marketplace_url = sys.argv[2] if len(sys.argv) > 2 else "https://www.amazon.com/"
|
||||
|
||||
result = run_amazon_listing_competitor_analysis_task(api_key, asin, marketplace_url)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,98 @@
|
||||
---
|
||||
name: amazon-product-api-skill
|
||||
description: "This skill helps users extract structured product listings from Amazon, including titles, ASINs, prices, ratings, and specifications. Use this skill when users want to search for products on Amazon, find the best selling brand products, track price changes for items, get a list of categories with high ratings, compare different brand products on Amazon, extract Amazon product data for market research, look for products in a specific language or marketplace, analyze competitor pricing for keywords, find featured products for search terms, get technical specifications like material or color for product lists."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Amazon Product Search Skill
|
||||
|
||||
## 📖 Introduction
|
||||
This skill utilizes BrowserAct's Amazon Product API template to extract structured product listings from Amazon search results. It provides detailed information including titles, ASINs, prices, ratings, and product specifications, enabling efficient market research and product monitoring without manual data collection.
|
||||
|
||||
## ✨ Features
|
||||
1. **No Hallucinations**: Pre-set workflows avoid AI generative hallucinations, ensuring stable and precise data extraction.
|
||||
2. **No Captcha Issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP Restrictions**: No need to handle regional IP restrictions or geofencing.
|
||||
4. **Faster Execution**: Tasks execute faster compared to pure AI-driven browser automation solutions.
|
||||
5. **Cost-Effective**: Significantly lowers data acquisition costs compared to high-token-consuming AI solutions.
|
||||
|
||||
## 🔑 API Key Setup
|
||||
Before running, check the `BROWSERACT_API_KEY` environment variable. If not set, do not take other measures; ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key, please visit the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
The agent should configure the following parameters based on user requirements:
|
||||
|
||||
1. **KeyWords**
|
||||
- **Type**: `string`
|
||||
- **Description**: Search keywords used to find products on Amazon.
|
||||
- **Required**: Yes
|
||||
- **Example**: `laptop`, `wireless earbuds`
|
||||
|
||||
2. **Brand**
|
||||
- **Type**: `string`
|
||||
- **Description**: Filter products by brand name.
|
||||
- **Default**: `Apple`
|
||||
- **Example**: `Dell`, `Samsung`
|
||||
|
||||
3. **Maximum_number_of_page_turns**
|
||||
- **Type**: `number`
|
||||
- **Description**: Number of search result pages to paginate through.
|
||||
- **Default**: `1`
|
||||
|
||||
4. **language**
|
||||
- **Type**: `string`
|
||||
- **Description**: UI language for the Amazon browsing session.
|
||||
- **Default**: `en`
|
||||
- **Example**: `zh-CN`, `de`
|
||||
|
||||
## 🚀 Usage
|
||||
Agent should use the following independent script to achieve "one-line command result":
|
||||
|
||||
```bash
|
||||
# Example Usage
|
||||
python -u ./scripts/amazon_product_api.py "keywords" "brand" pages "language"
|
||||
```
|
||||
|
||||
### ⏳ Execution Monitoring
|
||||
Since this task involves automated browser operations, it may take some time (several minutes). The script will **continuously output status logs with timestamps** (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent Instructions**:
|
||||
- While waiting for the script result, keep monitoring the terminal output.
|
||||
- As long as the terminal is outputting new status logs, the task is running normally; do not mistake it for a deadlock or unresponsiveness.
|
||||
- Only if the status remains unchanged for a long time or the script stops outputting without returning a result should you consider triggering the retry mechanism.
|
||||
|
||||
## 📊 Data Output
|
||||
Upon success, the script parses and prints the structured product data from the API response, which includes:
|
||||
- `product_title`: Full title of the product.
|
||||
- `asin`: Amazon Standard Identification Number.
|
||||
- `product_url`: URL of the Amazon product page.
|
||||
- `brand`: Brand name.
|
||||
- `price_current_amount`: Current price.
|
||||
- `price_original_amount`: Original price (if applicable).
|
||||
- `rating_average`: Average star rating.
|
||||
- `rating_count`: Total number of ratings.
|
||||
- `featured`: Badges like "Best Seller" or "Amazon's Choice".
|
||||
- `color`, `material`, `style`: Product attributes (if available).
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
If an error occurs during script execution (e.g., network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check Output Content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. **Do not retry**; guide the user to re-check and provide the correct API Key.
|
||||
- If the output **does not contain** `"Invalid authorization"` but the task failed (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically try to re-execute the script once**.
|
||||
|
||||
2. **Retry Limit**:
|
||||
- Automatic retry is limited to **one time**. If the second attempt fails, stop retrying and report the specific error information to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Market Research**: Search for a specific product category to analyze top brands and pricing.
|
||||
2. **Competitor Monitoring**: Track product listings and price changes for specific competitor brands.
|
||||
3. **Product Catalog Enrichment**: Extract structured details like ASINs and specifications to build or update a product database.
|
||||
4. **Rating Analysis**: Find high-rated products for specific keywords to identify market leaders.
|
||||
5. **Localized Research**: Search Amazon in different languages to analyze international markets.
|
||||
6. **Price Tracking**: Monitor current and original prices to identify discount trends.
|
||||
7. **Brand Performance**: Evaluate the presence of a specific brand in search results across multiple pages.
|
||||
8. **Attribute Extraction**: Gather technical specifications like material or color for a list of products.
|
||||
9. **Lead Generation**: Identify popular products and their manufacturers for business outreach.
|
||||
10. **Automated Data Feed**: Periodically pull Amazon search results into external BI tools or dashboards.
|
||||
@@ -0,0 +1,118 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams to handle multi-language content
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
# Amazon Product API Template ID
|
||||
TEMPLATE_ID = "77670107419143475"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
def run_amazon_product_task(api_key, keywords, brand="Apple", pages=1, language="en"):
|
||||
"""
|
||||
Executes an Amazon Product search task via BrowserAct API.
|
||||
"""
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "KeyWords", "value": keywords},
|
||||
{"name": "Brand", "value": brand},
|
||||
{"name": "Maximum_number_of_page_turns", "value": str(pages)},
|
||||
{"name": "language", "value": language}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Starting Amazon Product search task for keywords: {keywords}", flush=True)
|
||||
try:
|
||||
response = requests.post(f"{API_BASE_URL}/run-task-by-template", json=payload, headers=headers, timeout=30)
|
||||
res = response.json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
# Check for authorization error
|
||||
if "Invalid authorization" in str(res):
|
||||
print(f"Error: Invalid authorization. Please check your BrowserAct API Key.", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 300
|
||||
poll_start = time.time()
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(f"{API_BASE_URL}/get-task-status?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(f"{API_BASE_URL}/get-task?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
|
||||
# Extract data from output["string"] or the whole result
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Get API key from environment variable
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python amazon_product_api.py <keywords> [brand] [pages] [language]", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Set it as an environment variable (BROWSERACT_API_KEY) or provide it in the chat.", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
keywords = sys.argv[1]
|
||||
brand = sys.argv[2] if len(sys.argv) > 2 else "Apple"
|
||||
pages = sys.argv[3] if len(sys.argv) > 3 else 1
|
||||
language = sys.argv[4] if len(sys.argv) > 4 else "en"
|
||||
|
||||
result = run_amazon_product_task(api_key, keywords, brand, pages, language)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,97 @@
|
||||
---
|
||||
name: amazon-product-search-api-skill
|
||||
description: "This skill is designed to help users automatically extract product data from Amazon search results. The Agent should proactively apply this skill when users request searching for products related to keywords, finding best-selling items from specific brands, monitoring product prices and availability on Amazon, extracting product listings for market research, collecting product ratings and review counts for competitive analysis, finding specific products with a maximum count, searching Amazon in different languages for localized results, tracking monthly sales estimates for brand products, gathering product URLs and titles for a product catalog, scanning Amazon for Best Seller tags in a specific category, monitoring shipping and delivery information for brand items, building a structured dataset of Amazon search results."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Amazon Product Search Automation Skill
|
||||
|
||||
## 📖 Introduction
|
||||
This skill provides a one-stop product data collection service through BrowserAct's Amazon Product Search API template. It directly extracts structured product results from Amazon search lists. Simply input search keywords, brand filters, and quantity limits to get clean, usable product data.
|
||||
|
||||
## ✨ Features
|
||||
1. **No Hallucinations**: Pre-set workflows avoid AI generative hallucinations, ensuring stable and precise data extraction.
|
||||
2. **No Captcha Issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP Restrictions**: No need to handle regional IP restrictions or geofencing.
|
||||
4. **Faster Execution**: Tasks execute faster compared to pure AI-driven browser automation solutions.
|
||||
5. **Cost-Effective**: Significantly lowers data acquisition costs compared to high-token-consuming AI solutions.
|
||||
|
||||
## 🔑 API Key Setup
|
||||
Before running, check the `BROWSERACT_API_KEY` environment variable. If not set, do not take other measures; ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key, please visit the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
When calling the script, the Agent should flexibly configure the following parameters based on user needs:
|
||||
|
||||
1. **KeyWords (Search Keywords)**
|
||||
- **Type**: `string`
|
||||
- **Description**: The keywords the user wants to search for on Amazon.
|
||||
- **Example**: `phone`, `wireless earbuds`, `laptop stand`
|
||||
|
||||
2. **Brand (Brand Filter)**
|
||||
- **Type**: `string`
|
||||
- **Description**: Filter products by brand name shown in the listing.
|
||||
- **Example**: `Apple`, `Samsung`, `Sony`
|
||||
|
||||
3. **Maximum_date (Maximum Products)**
|
||||
- **Type**: `number`
|
||||
- **Description**: The maximum number of products to extract across paginated search results.
|
||||
- **Default**: `50`
|
||||
|
||||
4. **language (UI Language)**
|
||||
- **Type**: `string`
|
||||
- **Description**: UI language for the Amazon browsing session.
|
||||
- **Options**: `en`, `de`, `fr`, `it`, `es`, `ja`, `zh-CN`, `zh-TW`
|
||||
- **Default**: `en`
|
||||
|
||||
## 🚀 Usage
|
||||
The Agent should execute the following independent script to achieve "one-line command result":
|
||||
|
||||
```bash
|
||||
# Example Call
|
||||
python -u ./scripts/amazon_product_search_api.py "Keywords" "Brand" Quantity "language"
|
||||
```
|
||||
|
||||
### ⏳ Execution Monitoring
|
||||
Since this task involves automated browser operations, it may take some time (several minutes). The script will **continuously output status logs with timestamps** (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent Instructions**:
|
||||
- While waiting for the script result, keep monitoring the terminal output.
|
||||
- As long as the terminal is outputting new status logs, the task is running normally; do not mistake it for a deadlock or unresponsiveness.
|
||||
- Only if the status remains unchanged for a long time or the script stops outputting without returning a result should you consider triggering the retry mechanism.
|
||||
|
||||
## 📊 Data Output
|
||||
After successful execution, the script will parse and print results directly from the API response. Results include:
|
||||
- `product_title`: Product name
|
||||
- `product_url`: Detail page URL
|
||||
- `rating_score`: Average star rating
|
||||
- `review_count`: Total number of reviews
|
||||
- `monthly_sales`: Estimated monthly sales (if available)
|
||||
- `current_price`: Current selling price
|
||||
- `list_price`: Original list price (if available)
|
||||
- `delivery_info`: Delivery or fulfillment information
|
||||
- `shipping_location`: Shipping origin or location
|
||||
- `is_best_seller`: Whether marked as Best Seller
|
||||
- `is_available`: Whether available for purchase
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
If an error occurs during script execution (e.g., network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check Output Content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. **Do not retry**; guide the user to re-check and provide the correct API Key.
|
||||
- If the output **does not contain** `"Invalid authorization"` but the task failed (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically try to re-execute the script once**.
|
||||
|
||||
2. **Retry Limit**:
|
||||
- Automatic retry is limited to **one time**. If the second attempt fails, stop retrying and report the specific error information to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Market Research**: Search for "wireless earbuds" from "Sony" to analyze the current market.
|
||||
2. **Competitive Monitoring**: Track "Samsung" phone prices and availability on Amazon.
|
||||
3. **Catalog Discovery**: Gather product titles and URLs for a new product catalog in the "laptop stand" category.
|
||||
4. **Localized Analysis**: Search Amazon in "ja" (Japanese) to understand products available in the Japan region.
|
||||
5. **Best Seller Tracking**: Identify products marked as "Best Seller" for a specific brand.
|
||||
6. **Pricing Intelligence**: Compare `current_price` and `list_price` to monitor discounts.
|
||||
7. **Sales Trend Estimation**: Use `monthly_sales` data to estimate market demand for certain items.
|
||||
8. **Shipping Efficiency Study**: Analyze `delivery_info` and `shipping_location` for various brands.
|
||||
9. **Large-scale Data Extraction**: Collect up to 100 products for a comprehensive dataset.
|
||||
10. **Product Availability Check**: Verify if specific brand products are currently `is_available` for purchase.
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
TEMPLATE_ID = "77809217106347580"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
def run_amazon_product_search_task(api_key, keywords, brand="Apple", limit=50, language="en"):
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "KeyWords", "value": keywords},
|
||||
{"name": "Brand", "value": brand},
|
||||
{"name": "Maximum_date", "value": str(limit)},
|
||||
{"name": "language", "value": language}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Start Task", flush=True)
|
||||
try:
|
||||
res = requests.post(f"{API_BASE_URL}/run-task-by-template", json=payload, headers=headers, timeout=30).json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
# Check for authorization error
|
||||
if "Invalid authorization" in str(res):
|
||||
print(f"Error: Invalid authorization. Please check your BrowserAct API Key.", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 300
|
||||
poll_start = time.time()
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(f"{API_BASE_URL}/get-task-status?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(f"{API_BASE_URL}/get-task?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
|
||||
# Extract data from output["string"] as requested
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Prioritize command line API key, then environment variable
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python amazon_product_search_api.py <keywords> [brand] [limit] [language]", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Provide it to me or set it as an environment variable (BROWSERACT_API_KEY).", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
keywords = sys.argv[1]
|
||||
brand = sys.argv[2] if len(sys.argv) > 2 else "Apple"
|
||||
limit = sys.argv[3] if len(sys.argv) > 3 else 50
|
||||
language = sys.argv[4] if len(sys.argv) > 4 else "en"
|
||||
|
||||
result = run_amazon_product_search_task(api_key, keywords, brand, limit, language)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,79 @@
|
||||
---
|
||||
name: amazon-reviews-api-skill
|
||||
description: "This skill helps users automatically extract Amazon product reviews via the Amazon Reviews API. Agent should proactively apply this skill when users express needs like getting reviews for Amazon product with ASIN B07TS6R1SF, analyzing customer feedback for a specific Amazon item, getting ratings and comments for a competitive product, tracking sentiment of recent Amazon reviews, extracting verified purchase reviews for quality assessment, summarizing user experiences from Amazon product pages, monitoring product performance through customer reviews, collecting reviewer profiles and links for market research, gathering review titles and descriptions for content analysis, scraping Amazon reviews without requiring a login."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Amazon Reviews Automation Extraction Skill
|
||||
|
||||
## 📖 Introduction
|
||||
This skill provides a one-stop Amazon review collection service through BrowserAct's Amazon Reviews API template. It can directly extract structured review results from Amazon product pages. By simply providing an ASIN, you can get clean, usable review data without building crawler scripts or requiring an Amazon account login.
|
||||
|
||||
## ✨ Features
|
||||
1. **No Hallucinations**: Pre-set workflows avoid AI generative hallucinations, ensuring stable and precise data extraction.
|
||||
2. **No Captcha Issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP Restrictions**: No need to handle regional IP restrictions or geofencing.
|
||||
4. **Faster Execution**: Tasks execute faster compared to pure AI-driven browser automation solutions.
|
||||
5. **Cost-Effective**: Significantly lowers data acquisition costs compared to high-token-consuming AI solutions.
|
||||
|
||||
## 🔑 API Key Setup
|
||||
Before running, check the `BROWSERACT_API_KEY` environment variable. If not set, do not take other measures; ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key, please visit the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
When calling the script, the Agent should flexibly configure parameters based on user needs:
|
||||
|
||||
1. **ASIN (Amazon Standard Identification Number)**
|
||||
- **Type**: `string`
|
||||
- **Description**: The unique identifier for the product on Amazon.
|
||||
- **Example**: `B07TS6R1SF`, `B08N5WRWJ6`
|
||||
|
||||
## 🚀 Usage
|
||||
The Agent should execute the following independent script to achieve "one-line command result":
|
||||
|
||||
```bash
|
||||
# Example call
|
||||
python -u ./scripts/amazon_reviews_api.py "ASIN_HERE"
|
||||
```
|
||||
|
||||
### ⏳ Execution Monitoring
|
||||
Since this task involves automated browser operations, it may take some time (several minutes). The script will **continuously output status logs with timestamps** (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent Instructions**:
|
||||
- While waiting for the script result, keep monitoring the terminal output.
|
||||
- As long as the terminal is outputting new status logs, the task is running normally; do not mistake it for a deadlock or unresponsiveness.
|
||||
- Only if the status remains unchanged for a long time or the script stops outputting without returning a result should you consider triggering the retry mechanism.
|
||||
|
||||
## 📊 Data Output
|
||||
After successful execution, the script will parse and print results directly from the API response. Each review item includes:
|
||||
- `Commentator`: Reviewer's name
|
||||
- `Commenter profile link`: Link to the reviewer's profile
|
||||
- `Rating`: Star rating
|
||||
- `reviewTitle`: Headline of the review
|
||||
- `review Description`: Full text of the review
|
||||
- `Published at`: Date the review was published
|
||||
- `Country`: Reviewer's country
|
||||
- `Variant`: Product variant info (if available)
|
||||
- `Is Verified`: Whether it's a verified purchase
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
If an error occurs during script execution (e.g., network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check Output Content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. **Do not retry**; guide the user to re-check and provide the correct API Key.
|
||||
- If the output **does not contain** `"Invalid authorization"` but the task failed (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically try to re-execute the script once**.
|
||||
|
||||
2. **Retry Limit**:
|
||||
- Automatic retry is limited to **one time**. If the second attempt fails, stop retrying and report the specific error information to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Competitor Analysis**: Extract reviews for competitors' products to understand their strengths and weaknesses.
|
||||
2. **Product Feedback**: Summarize feedback for your own products to identify areas for improvement.
|
||||
3. **Market Research**: Collect data on customer preferences and common complaints in a specific category.
|
||||
4. **Sentiment Monitoring**: Monitor recent reviews to detect shifts in customer sentiment.
|
||||
5. **QA Insights**: Use customer reviews to identify potential quality issues or bugs.
|
||||
6. **Sentiment Analysis Prep**: Gather review text and ratings for detailed emotion modeling.
|
||||
7. **Verified Purchase Analysis**: Compare feedback from verified vs. unverified buyers.
|
||||
8. **Geographic Insights**: Analyze product performance across different reviewer countries.
|
||||
9. **Variant Comparison**: Understand which product variants (size/color) receive the best feedback.
|
||||
10. **Historical Trend Tracking**: Retrieve and analyze review publication dates to track product lifecycle sentiment.
|
||||
@@ -0,0 +1,105 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
TEMPLATE_ID = "77817507798321724"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
def run_amazon_reviews_task(api_key, asin):
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "ASIN", "value": asin}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Start Task", flush=True)
|
||||
try:
|
||||
res = requests.post(f"{API_BASE_URL}/run-task-by-template", json=payload, headers=headers, timeout=30).json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
# Check for authorization error
|
||||
if "Invalid authorization" in str(res):
|
||||
print(f"Error: Invalid authorization. Please check your BrowserAct API Key.", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 300
|
||||
poll_start = time.time()
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(f"{API_BASE_URL}/get-task-status?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(f"{API_BASE_URL}/get-task?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Prioritize environment variable
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python amazon_reviews_api.py <asin>", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Provide it to me or set it as an environment variable (BROWSERACT_API_KEY).", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
asin = sys.argv[1]
|
||||
|
||||
result = run_amazon_reviews_task(api_key, asin)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,162 @@
|
||||
---
|
||||
name: ecommerce-listing
|
||||
description: "Extract product list from any e-commerce category page, search results page, or keyword search with filters. Returns paginated product arrays with URL, name, price, currency, image, rating, review count per item. Supports URL input, keyword search, and site-scoped search with filters: price range, brand, category, minimum rating, in-stock only, and sort order. Works on Amazon, eBay, Walmart, Shopify collections, WooCommerce shops, Google Shopping, and any public product listing page. Use when: category listing, product search results, ecommerce search, search for products, filter products by price, list products from a site, price range filter, brand filter, keyword search with filters, scrape product list, product catalog extraction, get all products from category, bulk product URLs, product list scraping, category page scraper, search results scraper, multi-page product extraction."
|
||||
---
|
||||
|
||||
# E-commerce — Product Listing
|
||||
|
||||
> Category/search URL or keyword + filters → paginated product list (URL, name, price, image, rating per item)
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Extract a structured list of products from any e-commerce category, search results, or keyword search page, with support for price/brand/rating filters and multi-page pagination.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Target browser is open and connected
|
||||
- No login required for public listing pages
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. Use the bash tool for execution.
|
||||
|
||||
### DOM: Extract product list from current page
|
||||
|
||||
Navigate to the listing/search page first, then extract:
|
||||
|
||||
```bash
|
||||
eval "$(python scripts/extract-listing.py --max-results 20)"
|
||||
```
|
||||
|
||||
Parameters:
|
||||
- `--max-results`: max items to return per page, default 20
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"count": 20,
|
||||
"items": [
|
||||
{
|
||||
"url": "https://www.amazon.com/dp/B09WNK39JN",
|
||||
"name": "Amazon Echo Pop",
|
||||
"price": 39.99,
|
||||
"currency": "USD",
|
||||
"image": "https://m.media-amazon.com/images/I/...jpg",
|
||||
"rating": 4.7,
|
||||
"review_count": 103789,
|
||||
"asin": "B09WNK39JN"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### DOM: Get next page URL
|
||||
|
||||
After extracting a page, get the URL to navigate to for the next page:
|
||||
|
||||
```bash
|
||||
eval "$(python scripts/extract-listing-next-page.py)"
|
||||
```
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{"next_url": "https://www.amazon.com/s?k=headphones&page=2", "has_next": true, "method": "amazon"}
|
||||
```
|
||||
|
||||
When `has_next` is false, pagination is complete.
|
||||
|
||||
### Composite: Keyword search with filters → product list
|
||||
|
||||
**Step 1 — Build search URL with filters:**
|
||||
|
||||
Construct the URL based on target site and desired filters using the patterns below, then navigate:
|
||||
|
||||
**Amazon** (`amazon.com`):
|
||||
```
|
||||
https://www.amazon.com/s?k={keyword_urlencoded}&s={sort}&rh={filter_params}
|
||||
```
|
||||
- Sort (`s`): `price-asc-rank` | `price-desc-rank` | `review-rank` | `date-desc-rank` (omit for relevance)
|
||||
- Price filter: append `p_36:{min_cents}-{max_cents}` to `rh` (dollars × 100, e.g. $50–$200 → `p_36:5000-20000`)
|
||||
- Rating filter: append `avg_customer_review:four-and-above` | `three-and-above` | `two-and-above` to `rh`
|
||||
- In-stock: append `p_n_availability:1248801011` to `rh`
|
||||
- Multiple `rh` values: comma-separate (e.g. `rh=p_36:5000-20000,avg_customer_review:four-and-above`)
|
||||
|
||||
**eBay** (`ebay.com`):
|
||||
```
|
||||
https://www.ebay.com/sch/i.html?_nkw={keyword_urlencoded}&_udlo={min_price}&_udhi={max_price}&_sop={sort_num}
|
||||
```
|
||||
- Sort: `12`=BestMatch | `15`=PriceLow | `16`=PriceHigh | `24`=NewlyListed
|
||||
|
||||
**Walmart** (`walmart.com`):
|
||||
```
|
||||
https://www.walmart.com/search?q={keyword_urlencoded}&min_price={min}&max_price={max}&sort={sort}
|
||||
```
|
||||
- Sort: `best_match` | `price_low` | `price_high` | `rating_high`
|
||||
|
||||
**Google Shopping** (cross-site, no `--site`):
|
||||
```
|
||||
https://www.google.com/search?tbm=shop&q={keyword_urlencoded}&tbs=p_ord:{sort}
|
||||
```
|
||||
- Sort: `rv`=relevance | `pd`=price ascending | `prd`=price descending
|
||||
|
||||
**Any site with `--site`** (generic):
|
||||
```
|
||||
https://{site}/search?q={keyword_urlencoded}
|
||||
```
|
||||
|
||||
**Step 2 — Navigate and extract:**
|
||||
1. `navigate {constructed_url}` → `wait stable`
|
||||
2. `eval "$(python scripts/extract-listing.py --max-results {n})"`
|
||||
|
||||
**Step 3 — Paginate (repeat until done):**
|
||||
1. `eval "$(python scripts/extract-listing-next-page.py)"`
|
||||
2. If `has_next` is true: `navigate {next_url}` → `wait stable` → re-run extract-listing.py
|
||||
3. If `has_next` is false: stop
|
||||
|
||||
## Pagination
|
||||
|
||||
**URL Pagination**: `extract-listing-next-page.py` detects `rel=next` link, platform-specific pagination controls, and URL page parameters. Returns `next_url` for navigation.
|
||||
|
||||
**DOM Pagination**: For sites with load-more buttons (some Shopify themes):
|
||||
1. `state` to find "Load more" or "Show more" button
|
||||
2. `click <index>` → `wait stable` → re-run `extract-listing.py`
|
||||
3. Termination: button no longer present, or item count stops increasing
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`result.count >= 1 AND items[0].url != null`
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Amazon: direct navigation may trigger bot detection on fresh sessions — navigate from `https://www.amazon.com` first
|
||||
- eBay listing pages may require navigating from `https://www.ebay.com` first
|
||||
- Google Shopping results have complex SPA structure and may have reduced accuracy; prefer direct site search when `--site` is specified
|
||||
- Filter URL parameters are site-specific; unsupported filter parameters are silently ignored by some sites
|
||||
- Shopify themes vary widely; if the generic DOM strategies miss items, check if the page has JSON-LD ItemList or Product array in page source
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Loop through pages serially within a single session; add 1–2 second intervals between page navigations
|
||||
- **Test before batch execution**: Test with 1 page before running multi-page extraction
|
||||
- **Error resumption**: Record page number; on failure, resume from the last successful page
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/ecommerce-scraper-ecommerce-listing.memory.md`
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions; adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
@@ -0,0 +1,55 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.parse_args()
|
||||
|
||||
js = r"""
|
||||
(function() {
|
||||
try {
|
||||
// rel=next link (SEO-standard)
|
||||
const relNext = document.querySelector('link[rel="next"]')?.href;
|
||||
if (relNext) return JSON.stringify({ next_url: relNext, has_next: true, method: 'rel-next' });
|
||||
|
||||
// Amazon pagination
|
||||
const amzNext = document.querySelector('.a-pagination .a-last:not(.a-disabled) a')?.href;
|
||||
if (amzNext) return JSON.stringify({ next_url: amzNext, has_next: true, method: 'amazon' });
|
||||
|
||||
// eBay pagination
|
||||
const ebayNext = document.querySelector('.pagination__next a, a[aria-label="Go to next search page"]')?.href;
|
||||
if (ebayNext) return JSON.stringify({ next_url: ebayNext, has_next: true, method: 'ebay' });
|
||||
|
||||
// WooCommerce pagination
|
||||
const wooNext = document.querySelector('.woocommerce-pagination .next')?.href;
|
||||
if (wooNext) return JSON.stringify({ next_url: wooNext, has_next: true, method: 'woocommerce' });
|
||||
|
||||
// Generic next link patterns
|
||||
const genericNext = document.querySelector(
|
||||
'a[aria-label*="Next"], a[aria-label*="next"], .next-page a, .pagination .next a, [class*="pagination"] a[rel="next"], .pager .next a, a[class*="next-page"], a[class*="page-next"]'
|
||||
)?.href;
|
||||
if (genericNext) return JSON.stringify({ next_url: genericNext, has_next: true, method: 'generic' });
|
||||
|
||||
// URL-based page parameter increment
|
||||
const url = new URL(window.location.href);
|
||||
const pageKey = url.searchParams.has('page') ? 'page' : (url.searchParams.has('p') ? 'p' : (url.searchParams.has('pg') ? 'pg' : null));
|
||||
if (pageKey) {
|
||||
const nextPage = parseInt(url.searchParams.get(pageKey)) + 1;
|
||||
url.searchParams.set(pageKey, nextPage);
|
||||
const hasItems = document.querySelectorAll('[data-component-type="s-search-result"], .s-item, ul.products li.product, [class*="product-card"]').length > 0;
|
||||
if (hasItems) return JSON.stringify({ next_url: url.href, has_next: true, method: 'url-param' });
|
||||
}
|
||||
|
||||
return JSON.stringify({ next_url: null, has_next: false });
|
||||
} catch(e) {
|
||||
return JSON.stringify({ error: true, message: e.message });
|
||||
}
|
||||
})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,128 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--max-results', type=int, default=20)
|
||||
args = parser.parse_args()
|
||||
|
||||
js_template = r"""
|
||||
(function() {
|
||||
try {
|
||||
const maxResults = MAX_RESULTS;
|
||||
let items = [];
|
||||
|
||||
// Strategy 1: JSON-LD ItemList
|
||||
const lds = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map(s => {
|
||||
try { return JSON.parse(s.textContent); } catch(e) { return null; }
|
||||
}).filter(Boolean);
|
||||
const flat = lds.flatMap(l => Array.isArray(l) ? l : [l]);
|
||||
const listEl = flat.find(l => l['@type'] === 'ItemList' && l.itemListElement);
|
||||
if (listEl) {
|
||||
items = (listEl.itemListElement || []).slice(0, maxResults).map(e => {
|
||||
const item = e.item || e;
|
||||
const offer = Array.isArray(item.offers) ? item.offers[0] : item.offers;
|
||||
return { url: item.url, name: item.name, price: offer?.price != null ? parseFloat(offer.price) : null, currency: offer?.priceCurrency || null, image: Array.isArray(item.image) ? item.image[0] : (item.image || null), rating: item.aggregateRating?.ratingValue != null ? parseFloat(item.aggregateRating.ratingValue) : null, review_count: item.aggregateRating?.reviewCount != null ? parseInt(item.aggregateRating.reviewCount) : null };
|
||||
}).filter(item => item.url || item.name);
|
||||
}
|
||||
|
||||
// Strategy 2: Amazon search results
|
||||
if (items.length === 0) {
|
||||
const cards = Array.from(document.querySelectorAll('[data-component-type="s-search-result"]'));
|
||||
if (cards.length > 0) {
|
||||
items = cards.slice(0, maxResults).map(card => {
|
||||
const link = card.querySelector('h2 a, .a-link-normal.s-no-outline');
|
||||
const url = link?.href || null;
|
||||
const name = card.querySelector('h2 .a-text-normal, h2 span')?.textContent.trim() || null;
|
||||
const priceText = card.querySelector('.a-price .a-offscreen')?.textContent.trim();
|
||||
const price = priceText ? parseFloat(priceText.replace(/[^0-9.]/g, '')) : null;
|
||||
const currency = priceText?.match(/[A-Z]{3}/)?.[0] || (priceText?.includes('$') ? 'USD' : null);
|
||||
const image = card.querySelector('.s-image')?.src || null;
|
||||
const ratingText = card.querySelector('.a-icon-alt')?.textContent.trim();
|
||||
const rating = ratingText ? parseFloat(ratingText) : null;
|
||||
const rcText = card.querySelector('.a-size-small .a-link-normal')?.textContent.trim();
|
||||
const review_count = rcText ? parseInt(rcText.replace(/[^0-9]/g, '')) : null;
|
||||
const asin = card.getAttribute('data-asin') || null;
|
||||
return { url, name, price, currency, image, rating, review_count, asin };
|
||||
}).filter(item => item.url || item.name);
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 3: eBay search results
|
||||
if (items.length === 0) {
|
||||
const cards = Array.from(document.querySelectorAll('.s-item:not(.s-item--placeholder)'));
|
||||
if (cards.length > 0) {
|
||||
items = cards.slice(0, maxResults).map(card => {
|
||||
const link = card.querySelector('.s-item__link');
|
||||
const url = link?.href || null;
|
||||
const name = card.querySelector('.s-item__title')?.textContent.trim() || null;
|
||||
const priceText = card.querySelector('.s-item__price')?.textContent.trim();
|
||||
const price = priceText ? parseFloat(priceText.replace(/[^0-9.]/g, '')) : null;
|
||||
const image = card.querySelector('.s-item__image img')?.src || null;
|
||||
const condition = card.querySelector('.s-item__condition, .SECONDARY_INFO')?.textContent.trim() || null;
|
||||
const shipping = card.querySelector('.s-item__shipping, .s-item__logisticsCost')?.textContent.trim() || null;
|
||||
return { url, name, price, currency: null, image, condition, shipping };
|
||||
}).filter(item => item.url || item.name);
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 4: WooCommerce product grid
|
||||
if (items.length === 0) {
|
||||
const cards = Array.from(document.querySelectorAll('ul.products li.product'));
|
||||
if (cards.length > 0) {
|
||||
items = cards.slice(0, maxResults).map(card => {
|
||||
const link = card.querySelector('a.woocommerce-loop-product__link');
|
||||
const url = link?.href || null;
|
||||
const name = card.querySelector('.woocommerce-loop-product__title')?.textContent.trim() || null;
|
||||
const priceEl = card.querySelector('.price .woocommerce-Price-amount.amount');
|
||||
const price = priceEl ? parseFloat(priceEl.textContent.replace(/[^0-9.]/g, '')) : null;
|
||||
const image = card.querySelector('img.attachment-woocommerce_thumbnail, img.wp-post-image')?.src || null;
|
||||
const ratingEl = card.querySelector('.star-rating');
|
||||
const rating = ratingEl ? parseFloat(ratingEl.getAttribute('aria-label') || '') || null : null;
|
||||
return { url, name, price, currency: null, image, rating };
|
||||
}).filter(item => item.url || item.name);
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 5: Shopify collection — JSON-LD Product array (multiple products)
|
||||
if (items.length === 0) {
|
||||
const productLds = flat.filter(l => l['@type'] === 'Product');
|
||||
if (productLds.length > 1) {
|
||||
items = productLds.slice(0, maxResults).map(p => {
|
||||
const offer = Array.isArray(p.offers) ? p.offers[0] : p.offers;
|
||||
return { url: p.url || p['@id'] || null, name: p.name || null, price: offer?.price != null ? parseFloat(offer.price) : null, currency: offer?.priceCurrency || null, image: Array.isArray(p.image) ? p.image[0] : (p.image || null), rating: p.aggregateRating?.ratingValue != null ? parseFloat(p.aggregateRating.ratingValue) : null };
|
||||
}).filter(item => item.url || item.name);
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 6: Generic product cards heuristic
|
||||
if (items.length === 0) {
|
||||
const candidates = Array.from(document.querySelectorAll('article[class*="product"], [class*="product-card"], [class*="product-item"], [class*="product-tile"]'));
|
||||
const results = candidates.slice(0, maxResults).map(card => {
|
||||
const link = card.querySelector('a[href]');
|
||||
const name = card.querySelector('h2, h3, h4, [class*="title"], [class*="name"]')?.textContent.trim() || null;
|
||||
const priceText = card.querySelector('[class*="price"]')?.textContent.trim();
|
||||
const price = priceText ? parseFloat(priceText.replace(/[^0-9.]/g, '')) : null;
|
||||
const image = card.querySelector('img')?.src || null;
|
||||
return { url: link?.href || null, name, price, currency: null, image };
|
||||
}).filter(item => (item.url || item.name) && item.price != null);
|
||||
if (results.length > 0) items = results;
|
||||
}
|
||||
|
||||
if (items.length === 0) {
|
||||
return JSON.stringify({ error: true, message: 'No product listings found on this page. Ensure this is a category, search results, or product listing page.' });
|
||||
}
|
||||
return JSON.stringify({ count: items.length, items });
|
||||
} catch(e) {
|
||||
return JSON.stringify({ error: true, message: e.message });
|
||||
}
|
||||
})()
|
||||
"""
|
||||
js = js_template.replace('MAX_RESULTS', str(args.max_results))
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,123 @@
|
||||
---
|
||||
name: ecommerce-product-detail
|
||||
description: "Extract complete product information from any e-commerce product page. Returns name, price, currency, brand, images, description, SKU/ASIN/EAN/UPC/GTIN/MPN identifiers, stock availability, rating, review count, variants, and seller. Works on Shopify, Amazon, WooCommerce, eBay, Walmart, Etsy, AliExpress, Alibaba, Target, Best Buy, Rakuten, Magento, BigCommerce, PrestaShop, and any public e-commerce site. Accepts product URL, keyword, or product identifier (SKU/ASIN/EAN/UPC). Use when: scrape product page, get product details, extract price and availability, product info extraction, check product data, product detail scraping, get product from URL, keyword product search, ASIN lookup, EAN search, UPC lookup, price check, product research, compare products, monitor product price, get product images, product brand and description, ecommerce data extraction, product catalog scraping, product page scraper, get item price, fetch product info."
|
||||
---
|
||||
|
||||
# E-commerce — Product Detail
|
||||
|
||||
> Product URL / keyword / SKU → complete product data (name, price, brand, images, identifiers, availability, rating, variants)
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Extract complete product information from any publicly accessible e-commerce product page using a universal multi-layer extraction strategy (JSON-LD → platform-specific DOM → OG meta → microdata).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Target browser is open and connected
|
||||
- No login required for public product pages
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page, never bypassing authentication or access controls. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. Use the bash tool for execution.
|
||||
|
||||
### DOM: Extract product data from current product page
|
||||
|
||||
Navigate to the product URL first, then extract:
|
||||
|
||||
```bash
|
||||
eval "$(python scripts/extract-product.py)"
|
||||
```
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"url": "https://www.amazon.com/dp/B09WNK39JN",
|
||||
"name": "Amazon Echo Pop",
|
||||
"price": 39.99,
|
||||
"price_currency": "USD",
|
||||
"brand": "Amazon",
|
||||
"image": "https://m.media-amazon.com/images/I/61bTwy0ooPL.jpg",
|
||||
"images": ["https://...jpg", "https://...jpg"],
|
||||
"description": "Compact smart speaker with Alexa...",
|
||||
"category": ["Electronics", "Smart Speakers"],
|
||||
"sku": "B09WNK39JN",
|
||||
"gtin": null,
|
||||
"mpn": null,
|
||||
"availability": "InStock",
|
||||
"rating": 4.7,
|
||||
"review_count": 103789,
|
||||
"variants": [{"name": "Charcoal", "sku": "B09WNK39JN", "price": 39.99}],
|
||||
"seller": "Amazon",
|
||||
"identifiers": {"ASIN": "B09WNK39JN", "Best Sellers Rank": "#1 in Smart Speakers"},
|
||||
"_platform": "amazon",
|
||||
"_source": "json-ld"
|
||||
}
|
||||
```
|
||||
|
||||
### Composite: Keyword or SKU → product detail
|
||||
|
||||
When input is a keyword, ASIN/SKU, or EAN/UPC rather than a direct product URL:
|
||||
|
||||
**Step 1 — Navigate to search URL based on input type:**
|
||||
|
||||
| Input type | Target site | URL pattern |
|
||||
|-----------|-------------|-------------|
|
||||
| ASIN (10-char alphanumeric) | Amazon | `https://www.amazon.com/dp/{ASIN}` |
|
||||
| Keyword | Amazon | `https://www.amazon.com/s?k={keyword_urlencoded}` |
|
||||
| Keyword | eBay | `https://www.ebay.com/sch/i.html?_nkw={keyword_urlencoded}` |
|
||||
| Keyword | Walmart | `https://www.walmart.com/search?q={keyword_urlencoded}` |
|
||||
| Keyword + `--site` specified | Any site | `https://{site}/search?q={keyword_urlencoded}` |
|
||||
| Keyword (no site) | Cross-site | `https://www.google.com/search?tbm=shop&q={keyword_urlencoded}` |
|
||||
| EAN / UPC / GTIN | Cross-site | `https://www.google.com/search?tbm=shop&q={identifier}` |
|
||||
|
||||
**Step 2 — If landed on a search/listing page (multiple results):**
|
||||
1. `wait stable`
|
||||
2. `eval "$(python scripts/extract-listing.py --max-results 3)"` — get top 3 results
|
||||
3. Pick the most relevant product URL from `items[0].url`
|
||||
4. `navigate {product_url}` → `wait stable`
|
||||
|
||||
**Step 3 — Extract product data:**
|
||||
```bash
|
||||
eval "$(python scripts/extract-product.py)"
|
||||
```
|
||||
|
||||
Note: `scripts/extract-listing.py` is located in `../ecommerce-listing/scripts/extract-listing.py` if used as a standalone Skill install; otherwise reference the listing Skill.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`result.name != null AND (result.price != null OR result.availability != null)`
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Amazon bot detection: direct navigation to a product URL may redirect to a CAPTCHA or bot-check page on fresh sessions. Navigate from `https://www.amazon.com` first to establish session cookies, then navigate to the product page
|
||||
- eBay product pages may require navigating from `https://www.ebay.com` first; use `solve-captcha` if a challenge appears
|
||||
- Some sites render product data entirely via client-side JavaScript; always use `wait stable` before extracting
|
||||
- Price may be null for out-of-stock items or when login is required to view pricing
|
||||
- Variant data completeness depends on whether the site includes full variant markup in JSON-LD
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script to loop through product URLs serially within a single session; add 1–2 second intervals between requests to avoid triggering anti-scraping restrictions
|
||||
- **Test before batch execution**: Test with 1–2 URLs before running the full batch
|
||||
- **Error resumption**: Save results item by item; on failure, resume from the breakpoint rather than starting over
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/ecommerce-scraper-ecommerce-product-detail.memory.md`
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
@@ -0,0 +1,157 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.parse_args()
|
||||
|
||||
js = r"""
|
||||
(function() {
|
||||
try {
|
||||
const result = {};
|
||||
|
||||
// Layer 1: JSON-LD (schema.org/Product)
|
||||
const lds = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map(s => {
|
||||
try { return JSON.parse(s.textContent); } catch(e) { return null; }
|
||||
}).filter(Boolean);
|
||||
const flat = lds.flatMap(l => Array.isArray(l) ? l : [l]);
|
||||
const pld = flat.find(l => l['@type'] === 'Product');
|
||||
const rld = flat.find(l => l['@type'] === 'AggregateRating' && l.itemReviewed?.['@type'] === 'Product');
|
||||
|
||||
if (pld) {
|
||||
result.name = pld.name || null;
|
||||
result.description = pld.description || null;
|
||||
result.brand = (typeof pld.brand === 'string' ? pld.brand : pld.brand?.name) || null;
|
||||
const imgs = Array.isArray(pld.image) ? pld.image : (pld.image ? [pld.image] : []);
|
||||
result.image = imgs[0] || null;
|
||||
result.images = imgs.length > 0 ? imgs : null;
|
||||
result.category = Array.isArray(pld.category) ? pld.category : (pld.category ? [pld.category] : null);
|
||||
result.sku = pld.sku || null;
|
||||
result.gtin = pld.gtin || pld.gtin14 || pld.gtin13 || pld.gtin12 || pld.gtin8 || null;
|
||||
result.mpn = pld.mpn || null;
|
||||
result.isbn = pld.isbn || null;
|
||||
const offer = Array.isArray(pld.offers) ? pld.offers[0] : pld.offers;
|
||||
if (offer) {
|
||||
result.price = offer.price != null ? parseFloat(offer.price) : null;
|
||||
result.price_currency = offer.priceCurrency || null;
|
||||
result.availability = offer.availability ? offer.availability.replace('https://schema.org/', '') : null;
|
||||
result.seller = (typeof offer.seller === 'string' ? offer.seller : offer.seller?.name) || null;
|
||||
}
|
||||
const agg = pld.aggregateRating || rld;
|
||||
if (agg) {
|
||||
result.rating = agg.ratingValue != null ? parseFloat(agg.ratingValue) : null;
|
||||
result.review_count = agg.reviewCount != null ? parseInt(agg.reviewCount) : (agg.ratingCount != null ? parseInt(agg.ratingCount) : null);
|
||||
}
|
||||
if (pld.hasVariant) {
|
||||
const vars = Array.isArray(pld.hasVariant) ? pld.hasVariant : [pld.hasVariant];
|
||||
result.variants = vars.slice(0, 30).map(v => ({ name: v.name, sku: v.sku, price: v.offers?.price != null ? parseFloat(v.offers.price) : null }));
|
||||
}
|
||||
result._source = 'json-ld';
|
||||
}
|
||||
|
||||
// Layer 2a: Shopify — window.ShopifyAnalytics.meta.product
|
||||
if (window.ShopifyAnalytics?.meta?.product) {
|
||||
const sp = window.ShopifyAnalytics.meta.product;
|
||||
if (!result.brand) result.brand = sp.vendor || null;
|
||||
if (!result.category && sp.type) result.category = [sp.type];
|
||||
if (!result.sku && sp.variants?.[0]?.sku) result.sku = sp.variants[0].sku;
|
||||
if (!result.variants && sp.variants?.length > 0) {
|
||||
result.variants = sp.variants.slice(0, 30).map(v => ({ name: v.public_title || v.name, sku: v.sku, price: v.price / 100 }));
|
||||
}
|
||||
result._platform = 'shopify';
|
||||
}
|
||||
|
||||
// Layer 2b: Amazon DOM
|
||||
if (window.location.hostname.includes('amazon.')) {
|
||||
const asin = window.location.pathname.match(/\/dp\/([A-Z0-9]{10})/)?.[1];
|
||||
if (!result.name) result.name = document.querySelector('#productTitle')?.textContent.trim() || null;
|
||||
if (!result.price) {
|
||||
const pt = document.querySelector('.a-price .a-offscreen')?.textContent.trim();
|
||||
if (pt) { result.price = parseFloat(pt.replace(/[^0-9.]/g, '')); result.price_currency = pt.match(/[A-Z]{3}/)?.[0] || (pt.includes('$') ? 'USD' : null); }
|
||||
}
|
||||
if (!result.brand) result.brand = document.querySelector('#bylineInfo')?.textContent.trim().replace(/^(Brand:|Visit the|by\s+)/i, '').trim() || null;
|
||||
if (!result.rating) {
|
||||
const rt = document.querySelector('#acrPopover')?.getAttribute('title') || '';
|
||||
result.rating = rt ? parseFloat(rt) : null;
|
||||
}
|
||||
if (!result.review_count) {
|
||||
const rc = document.querySelector('#acrCustomerReviewText')?.textContent.trim();
|
||||
result.review_count = rc ? parseInt(rc.replace(/[^0-9]/g, '')) : null;
|
||||
}
|
||||
if (!result.availability) result.availability = document.querySelector('#availability span')?.textContent.trim() || null;
|
||||
if (!result.image) result.image = document.querySelector('#landingImage')?.src || document.querySelector('#imgBlkFront')?.src || null;
|
||||
if (!result.images || result.images.length === 0) {
|
||||
result.images = Array.from(document.querySelectorAll('.imageThumbnail img, #altImgTmb img')).map(img => img.src.replace(/\._[A-Z0-9_,]+_\./i, '._SX522_.')).filter(s => s.length > 20 && !s.includes('grey-pixel')).slice(0, 10);
|
||||
}
|
||||
if (!result.description) result.description = Array.from(document.querySelectorAll('#feature-bullets .a-list-item')).map(el => el.textContent.trim()).filter(t => t.length > 5).join('\n') || null;
|
||||
const details = {};
|
||||
document.querySelectorAll('#detailBulletsWrapper_feature_div li').forEach(li => {
|
||||
const spans = li.querySelectorAll('span');
|
||||
if (spans.length >= 2) { const k = spans[0].textContent.replace(':', '').trim(); const v = spans[1]?.textContent.trim(); if (k && v && k.length < 60) details[k] = v; }
|
||||
});
|
||||
document.querySelectorAll('#productDetails_techSpec_section_1 tr, #productDetails_detailBullets_sections1 tr').forEach(row => {
|
||||
const k = row.querySelector('th')?.textContent.trim(); const v = row.querySelector('td')?.textContent.trim().replace(/\s+/g, ' ');
|
||||
if (k && v) details[k] = v;
|
||||
});
|
||||
if (asin) { details['ASIN'] = asin; if (!result.sku) result.sku = asin; }
|
||||
if (Object.keys(details).length > 0) result.identifiers = details;
|
||||
if (!result._platform) result._platform = 'amazon';
|
||||
}
|
||||
|
||||
// Layer 2c: WooCommerce DOM
|
||||
const wooPrice = document.querySelector('.price .woocommerce-Price-amount.amount');
|
||||
if (wooPrice && !result._platform) {
|
||||
if (!result.name) result.name = document.querySelector('.product_title, h1.product_title')?.textContent.trim() || null;
|
||||
if (!result.price) result.price = parseFloat(wooPrice.textContent.replace(/[^0-9.]/g, '')) || null;
|
||||
if (!result.sku) result.sku = document.querySelector('.sku')?.textContent.trim() || null;
|
||||
if (!result.description) result.description = document.querySelector('.woocommerce-product-details__short-description')?.textContent.trim() || null;
|
||||
if (!result.category) result.category = Array.from(document.querySelectorAll('.posted_in a')).map(a => a.textContent.trim());
|
||||
if (!result.availability) result.availability = document.querySelector('.stock')?.textContent.trim() || null;
|
||||
if (!result.rating) {
|
||||
const stars = document.querySelector('.star-rating')?.getAttribute('aria-label');
|
||||
result.rating = stars ? parseFloat(stars) : null;
|
||||
}
|
||||
if (!result.review_count) {
|
||||
const rc = document.querySelector('.woocommerce-review-link')?.textContent.trim().match(/\d+/)?.[0];
|
||||
result.review_count = rc ? parseInt(rc) : null;
|
||||
}
|
||||
result._platform = 'woocommerce';
|
||||
}
|
||||
|
||||
// Layer 3: OG meta fallback
|
||||
if (!result.name) result.name = document.querySelector('meta[property="og:title"]')?.getAttribute('content') || null;
|
||||
if (!result.image) result.image = document.querySelector('meta[property="og:image"]')?.getAttribute('content') || null;
|
||||
if (!result.description) result.description = document.querySelector('meta[property="og:description"], meta[name="description"]')?.getAttribute('content') || null;
|
||||
if (!result.price) {
|
||||
const ogp = document.querySelector('meta[property="og:price:amount"]')?.getAttribute('content');
|
||||
if (ogp) result.price = parseFloat(ogp);
|
||||
}
|
||||
if (!result.price_currency) result.price_currency = document.querySelector('meta[property="og:price:currency"]')?.getAttribute('content') || null;
|
||||
|
||||
// Layer 4: Microdata fallback
|
||||
if (!result.name) result.name = document.querySelector('[itemprop="name"]')?.textContent.trim() || null;
|
||||
if (!result.price) {
|
||||
const mp = document.querySelector('[itemprop="price"]');
|
||||
if (mp) result.price = parseFloat(mp.getAttribute('content') || mp.textContent.replace(/[^0-9.]/g, '')) || null;
|
||||
}
|
||||
if (!result.sku) result.sku = document.querySelector('[itemprop="sku"]')?.textContent.trim() || document.querySelector('[itemprop="sku"]')?.getAttribute('content') || null;
|
||||
if (!result.brand) result.brand = document.querySelector('[itemprop="brand"] [itemprop="name"], [itemprop="brand"]')?.textContent.trim() || null;
|
||||
|
||||
result.url = window.location.href;
|
||||
|
||||
if (!result.name && !result.price) {
|
||||
return JSON.stringify({ error: true, message: 'No product data found. Ensure this is a product detail page and the page has fully loaded.' });
|
||||
}
|
||||
return JSON.stringify(result);
|
||||
} catch(e) {
|
||||
return JSON.stringify({ error: true, message: e.message });
|
||||
}
|
||||
})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,122 @@
|
||||
---
|
||||
name: ecommerce-reviews
|
||||
description: "Extract customer reviews from any e-commerce product page or reviews page. Returns reviewer name, star rating, date, review title, review body, verified purchase status, and helpful votes per review. Works on Amazon, WooCommerce, Shopify, and any site with standard review markup. Supports pagination for multi-page review sections. Use when: product reviews, customer feedback, review scraping, get reviews, sentiment analysis data, review extraction, customer ratings, extract customer opinions, product feedback, user reviews, review mining, bulk review collection, review analysis, scrape ratings and comments, ecommerce review data."
|
||||
---
|
||||
|
||||
# E-commerce — Product Reviews
|
||||
|
||||
> Product URL → paginated customer reviews (reviewer, rating, date, title, body, verified, helpful votes)
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Extract customer reviews from any publicly accessible e-commerce product or reviews page using a multi-strategy approach (JSON-LD Review → Amazon DOM → WooCommerce DOM → generic microdata → generic CSS patterns).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Target browser is open and connected
|
||||
- No login required for public review pages
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. Use the bash tool for execution.
|
||||
|
||||
### DOM: Extract reviews from current page
|
||||
|
||||
Navigate to the product/reviews page first, then extract:
|
||||
|
||||
```bash
|
||||
eval "$(python scripts/extract-reviews.py --max-reviews 20)"
|
||||
```
|
||||
|
||||
Parameters:
|
||||
- `--max-reviews`: max reviews to return per page, default 20
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"count": 20,
|
||||
"reviews": [
|
||||
{
|
||||
"reviewer": "John D.",
|
||||
"rating": 5.0,
|
||||
"date": "Reviewed in the United States on May 15, 2026",
|
||||
"title": "Great product, exactly as described",
|
||||
"body": "I've been using this for two weeks and it works perfectly...",
|
||||
"verified": true,
|
||||
"helpful_votes": 42
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Composite: Product URL → reviews with sort and pagination
|
||||
|
||||
**Step 1 — Navigate to reviews page:**
|
||||
|
||||
| Platform | Reviews URL pattern |
|
||||
|----------|---------------------|
|
||||
| Amazon | `https://www.amazon.com/product-reviews/{ASIN}?sortBy=recent` (most recent) or `sortBy=helpful` |
|
||||
| Amazon (from product page) | Scroll to reviews section or click "See all reviews" link, `wait stable` |
|
||||
| WooCommerce | Product page URL with `#reviews` anchor; reviews are inline on the page |
|
||||
| Shopify | Reviews are typically inline on the product page |
|
||||
| Generic | Navigate to product URL; reviews section is usually below product info |
|
||||
|
||||
**Step 2 — Extract reviews:**
|
||||
```bash
|
||||
eval "$(python scripts/extract-reviews.py --max-reviews 20)"
|
||||
```
|
||||
|
||||
**Step 3 — Paginate (Amazon):**
|
||||
Amazon review pages support URL pagination:
|
||||
- Most recent sort: `https://www.amazon.com/product-reviews/{ASIN}?sortBy=recent&pageNumber={page}`
|
||||
- Helpful sort: `https://www.amazon.com/product-reviews/{ASIN}?sortBy=helpful&pageNumber={page}`
|
||||
|
||||
For each page: `navigate {reviews_url_with_page}` → `wait stable` → re-run extract-reviews.py
|
||||
|
||||
Termination: when `count` returns 0, or no new reviews appear compared to prior page.
|
||||
|
||||
## Pagination
|
||||
|
||||
**URL Pagination (Amazon)**: Increment `pageNumber` parameter in the reviews URL. Start from 1.
|
||||
|
||||
**DOM Pagination (WooCommerce/generic)**: Look for a "Next" pagination link on the reviews section. Use `eval "$(python ../ecommerce-listing/scripts/extract-listing-next-page.py)"` to detect it, then navigate.
|
||||
|
||||
Termination: `has_next` is false, or `count` is 0.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`result.count >= 1 AND reviews[0].body != null`
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Amazon: navigate from `https://www.amazon.com` first on fresh sessions to avoid bot detection
|
||||
- JSON-LD reviews are often limited to a small subset (3–5 reviews) even when hundreds exist; use the Amazon-specific URL for full review extraction
|
||||
- WooCommerce and Shopify review data depends on which review plugin is installed; body extraction may be null if a non-standard plugin is used
|
||||
- Review dates may be locale-formatted strings rather than ISO dates depending on the site's configuration
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Loop through review pages serially; add 1–2 second intervals between navigations
|
||||
- **Test before batch execution**: Test with page 1 before running multi-page extraction
|
||||
- **Error resumption**: Record page number; on failure, resume from last successful page
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/ecommerce-scraper-ecommerce-reviews.memory.md`
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions; adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
@@ -0,0 +1,124 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--max-reviews', type=int, default=20)
|
||||
args = parser.parse_args()
|
||||
|
||||
js_template = r"""
|
||||
(function() {
|
||||
try {
|
||||
const maxReviews = MAX_REVIEWS;
|
||||
let reviews = [];
|
||||
|
||||
// Strategy 1: JSON-LD Review[]
|
||||
const lds = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map(s => {
|
||||
try { return JSON.parse(s.textContent); } catch(e) { return null; }
|
||||
}).filter(Boolean);
|
||||
const flat = lds.flatMap(l => Array.isArray(l) ? l : [l]);
|
||||
const pld = flat.find(l => l['@type'] === 'Product' && l.review);
|
||||
if (pld?.review) {
|
||||
const ldRevs = Array.isArray(pld.review) ? pld.review : [pld.review];
|
||||
reviews = ldRevs.slice(0, maxReviews).map(r => ({
|
||||
reviewer: r.author?.name || (typeof r.author === 'string' ? r.author : null),
|
||||
rating: r.reviewRating?.ratingValue != null ? parseFloat(r.reviewRating.ratingValue) : null,
|
||||
date: r.datePublished || null,
|
||||
title: r.name || null,
|
||||
body: r.reviewBody || null,
|
||||
verified: null,
|
||||
helpful_votes: null
|
||||
}));
|
||||
}
|
||||
|
||||
// Strategy 2: Amazon reviews
|
||||
if (reviews.length === 0) {
|
||||
const amzCards = Array.from(document.querySelectorAll('[data-hook="review"]'));
|
||||
if (amzCards.length > 0) {
|
||||
reviews = amzCards.slice(0, maxReviews).map(card => {
|
||||
const ratingText = card.querySelector('[data-hook="review-star-rating"] .a-icon-alt, [data-hook="cmps-review-star-rating"] .a-icon-alt')?.textContent.trim();
|
||||
const helpfulText = card.querySelector('[data-hook="helpful-vote-statement"]')?.textContent.trim();
|
||||
return {
|
||||
reviewer: card.querySelector('.a-profile-name')?.textContent.trim() || null,
|
||||
rating: ratingText ? parseFloat(ratingText) : null,
|
||||
date: card.querySelector('[data-hook="review-date"]')?.textContent.trim() || null,
|
||||
title: card.querySelector('[data-hook="review-title"] span:not([class])')?.textContent.trim() || null,
|
||||
body: card.querySelector('[data-hook="review-body"] span')?.textContent.trim() || null,
|
||||
verified: !!card.querySelector('[data-hook="avp-badge"]'),
|
||||
helpful_votes: helpfulText ? parseInt(helpfulText.match(/\d+/)?.[0] || '0') : 0
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 3: WooCommerce reviews
|
||||
if (reviews.length === 0) {
|
||||
const wooCards = Array.from(document.querySelectorAll('.woocommerce-Reviews .review, ol.commentlist li'));
|
||||
if (wooCards.length > 0) {
|
||||
reviews = wooCards.slice(0, maxReviews).map(card => {
|
||||
const ratingEl = card.querySelector('.star-rating');
|
||||
const ratingAria = ratingEl?.getAttribute('aria-label');
|
||||
const rating = ratingAria ? parseFloat(ratingAria) : null;
|
||||
return {
|
||||
reviewer: card.querySelector('.reviewer, .woocommerce-review__author')?.textContent.trim() || null,
|
||||
rating,
|
||||
date: card.querySelector('[datetime]')?.getAttribute('datetime') || card.querySelector('time')?.textContent.trim() || null,
|
||||
title: null,
|
||||
body: card.querySelector('.description p, .comment-text p')?.textContent.trim() || null,
|
||||
verified: !!card.querySelector('.woocommerce-review__verified'),
|
||||
helpful_votes: null
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 4: Generic [itemprop="review"]
|
||||
if (reviews.length === 0) {
|
||||
const genericCards = Array.from(document.querySelectorAll('[itemprop="review"]'));
|
||||
if (genericCards.length > 0) {
|
||||
reviews = genericCards.slice(0, maxReviews).map(card => {
|
||||
const ratingEl = card.querySelector('[itemprop="ratingValue"]');
|
||||
return {
|
||||
reviewer: card.querySelector('[itemprop="author"]')?.textContent.trim() || null,
|
||||
rating: ratingEl ? parseFloat(ratingEl.getAttribute('content') || ratingEl.textContent) : null,
|
||||
date: card.querySelector('[itemprop="datePublished"]')?.getAttribute('content') || card.querySelector('[itemprop="datePublished"]')?.textContent.trim() || null,
|
||||
title: card.querySelector('[itemprop="name"]')?.textContent.trim() || null,
|
||||
body: card.querySelector('[itemprop="description"], [itemprop="reviewBody"]')?.textContent.trim() || null,
|
||||
verified: null,
|
||||
helpful_votes: null
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 5: Generic .review containers
|
||||
if (reviews.length === 0) {
|
||||
const cards = Array.from(document.querySelectorAll('.review-item, .review-card, .product-review'));
|
||||
reviews = cards.slice(0, maxReviews).map(card => ({
|
||||
reviewer: card.querySelector('[class*="name"], [class*="author"], [class*="user"]')?.textContent.trim() || null,
|
||||
rating: null,
|
||||
date: card.querySelector('time, [class*="date"]')?.textContent.trim() || null,
|
||||
title: card.querySelector('h3, h4, [class*="title"]')?.textContent.trim() || null,
|
||||
body: card.querySelector('p, [class*="body"], [class*="text"]')?.textContent.trim() || null,
|
||||
verified: null,
|
||||
helpful_votes: null
|
||||
})).filter(r => r.body);
|
||||
}
|
||||
|
||||
if (reviews.length === 0) {
|
||||
return JSON.stringify({ error: true, message: 'No reviews found on this page. Navigate to the product reviews section or reviews page first.' });
|
||||
}
|
||||
return JSON.stringify({ count: reviews.length, reviews });
|
||||
} catch(e) {
|
||||
return JSON.stringify({ error: true, message: e.message });
|
||||
}
|
||||
})()
|
||||
"""
|
||||
js = js_template.replace('MAX_REVIEWS', str(args.max_reviews))
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,111 @@
|
||||
---
|
||||
name: ecommerce-seller-info
|
||||
description: "Extract seller or merchant profile data from marketplace platform seller pages. Returns seller name, rating, review count, positive feedback percentage, joined date, and return policy. Works on Amazon seller pages, eBay seller pages, and any e-commerce site with seller profiles. Use when: seller information, merchant profile, seller rating, marketplace seller data, seller details, vendor profile, store information, seller feedback, merchant rating, seller review count, get seller info, ebay seller page, amazon seller profile, marketplace vendor analysis, seller research."
|
||||
---
|
||||
|
||||
# E-commerce — Seller Info
|
||||
|
||||
> Seller/merchant profile URL → seller name, rating, review count, feedback, joined date, return policy
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Extract seller profile information from marketplace platform seller or storefront pages using JSON-LD structured data and platform-specific DOM patterns.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Target browser is open and connected
|
||||
- No login required for public seller profile pages
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. Use the bash tool for execution.
|
||||
|
||||
### DOM: Extract seller profile from current seller page
|
||||
|
||||
Navigate to the seller profile URL first, then extract:
|
||||
|
||||
```bash
|
||||
eval "$(python scripts/extract-seller.py)"
|
||||
```
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"url": "https://www.amazon.com/shops/seller/A1234567890",
|
||||
"name": "TechGadgets Store",
|
||||
"description": "Premium electronics accessories since 2015",
|
||||
"rating": 4.8,
|
||||
"review_count": 12450,
|
||||
"positive_feedback_pct": "98% positive feedback",
|
||||
"joined": "Member since: January 2015",
|
||||
"return_policy": "30-day returns accepted",
|
||||
"image": null,
|
||||
"_platform": "amazon"
|
||||
}
|
||||
```
|
||||
|
||||
### Composite: Amazon seller URL patterns
|
||||
|
||||
Amazon seller pages follow these URL patterns:
|
||||
|
||||
| Seller page type | URL |
|
||||
|-----------------|-----|
|
||||
| Seller storefront | `https://www.amazon.com/shops/{seller_id}` |
|
||||
| Seller feedback (from product page) | Click "Sold by {seller_name}" link on a product page |
|
||||
| Third-party seller ratings | `https://www.amazon.com/gp/seller/{seller_id}/ref=dp_byline_sr` |
|
||||
|
||||
To find a seller from a product page:
|
||||
1. Navigate to product page → `wait stable`
|
||||
2. `eval "document.querySelector('#sellerProfileTriggerId, #merchant-info a')?.href"` to get the seller URL
|
||||
3. `navigate {seller_url}` → `wait stable`
|
||||
4. `eval "$(python scripts/extract-seller.py)"`
|
||||
|
||||
### Composite: eBay seller URL patterns
|
||||
|
||||
| Seller page type | URL |
|
||||
|-----------------|-----|
|
||||
| eBay seller storefront | `https://www.ebay.com/str/{seller_username}` |
|
||||
| eBay seller feedback | `https://www.ebay.com/usr/{seller_username}` |
|
||||
|
||||
To find seller from an eBay listing:
|
||||
1. Navigate to eBay item page → `wait stable`
|
||||
2. `eval "document.querySelector('.x-sellercard-atf__data a[href*=\"/usr/\"]')?.href"` to get seller URL
|
||||
3. Navigate and extract
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`result.name != null`
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Amazon seller pages may require navigating from `https://www.amazon.com` first on fresh sessions to avoid bot detection
|
||||
- eBay seller pages may require navigating from `https://www.ebay.com` first
|
||||
- Seller description and return policy availability depends on whether the seller has filled in their profile
|
||||
- Rating scale differs by platform: Amazon uses 1–5 stars, eBay uses percentage of positive feedback; both are preserved in their native format
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Loop through seller URLs serially; add 1–2 second intervals between navigations
|
||||
- **Test before batch execution**: Test with 1–2 sellers before running the full batch
|
||||
- **Error resumption**: Save results item by item; on failure, resume from the breakpoint
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/ecommerce-scraper-ecommerce-seller-info.memory.md`
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions; adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
@@ -0,0 +1,89 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.parse_args()
|
||||
|
||||
js = r"""
|
||||
(function() {
|
||||
try {
|
||||
const result = {};
|
||||
|
||||
// Strategy 1: JSON-LD Person/Organization/LocalBusiness/Store
|
||||
const lds = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map(s => {
|
||||
try { return JSON.parse(s.textContent); } catch(e) { return null; }
|
||||
}).filter(Boolean);
|
||||
const flat = lds.flatMap(l => Array.isArray(l) ? l : [l]);
|
||||
const sellerLd = flat.find(l => ['Person', 'Organization', 'LocalBusiness', 'Store', 'OnlineStore'].includes(l['@type']));
|
||||
if (sellerLd) {
|
||||
result.name = sellerLd.name || null;
|
||||
result.description = sellerLd.description || null;
|
||||
result.url = sellerLd.url || null;
|
||||
result.image = Array.isArray(sellerLd.image) ? sellerLd.image[0] : (sellerLd.image || null);
|
||||
result.rating = sellerLd.aggregateRating?.ratingValue != null ? parseFloat(sellerLd.aggregateRating.ratingValue) : null;
|
||||
result.review_count = sellerLd.aggregateRating?.reviewCount != null ? parseInt(sellerLd.aggregateRating.reviewCount) : null;
|
||||
result._source = 'json-ld';
|
||||
}
|
||||
|
||||
// Strategy 2: Amazon seller page
|
||||
if (window.location.hostname.includes('amazon.')) {
|
||||
// Try specific seller/storefront selectors before falling back to generic h1
|
||||
const sellerNameEl = document.querySelector('#sellerName, .a-profile-name.seller-name, #storeName, [class*="storeName"], .store-name-text, .s-store-name');
|
||||
// Also try: first h1 inside the main seller content container (not cart widgets)
|
||||
const contentH1 = document.querySelector('#dp-container h1, #storefront-header h1, .seller-profile-container h1, [data-cel-widget="merchant-info"] h1');
|
||||
const sellerName = (sellerNameEl || contentH1)?.textContent.trim();
|
||||
if (!result.name && sellerName) result.name = sellerName;
|
||||
const ratingText = document.querySelector('#seller-feedback-summary .feedback-count-text, .feedback-detail')?.textContent.trim();
|
||||
if (!result.rating && ratingText) {
|
||||
const m = ratingText.match(/(\d+\.?\d*)\s*out of\s*5/i);
|
||||
if (m) result.rating = parseFloat(m[1]);
|
||||
}
|
||||
const rcText = document.querySelector('#seller-feedback-summary .feedback-count, a[href*="feedback"]')?.textContent.trim();
|
||||
if (!result.review_count && rcText) result.review_count = parseInt(rcText.replace(/[^0-9]/g, '')) || null;
|
||||
result._platform = 'amazon';
|
||||
}
|
||||
|
||||
// Strategy 3: eBay seller
|
||||
if (window.location.hostname.includes('ebay.')) {
|
||||
const ebayName = document.querySelector('.str-seller-card__seller-name a, .si-content h1, .str-title')?.textContent.trim();
|
||||
if (!result.name && ebayName) result.name = ebayName;
|
||||
const posFeedback = document.querySelector('.str-seller-card__positive-feedback, .positive-feedback-percentage')?.textContent.trim();
|
||||
if (posFeedback) result.positive_feedback_pct = posFeedback;
|
||||
const fbCount = document.querySelector('.str-seller-card__feedback-count, .feedback-number')?.textContent.trim();
|
||||
if (!result.review_count && fbCount) result.review_count = parseInt(fbCount.replace(/[^0-9]/g, '')) || null;
|
||||
result._platform = 'ebay';
|
||||
}
|
||||
|
||||
// Strategy 4: Generic fallback
|
||||
if (!result.name) result.name = document.querySelector('h1, .seller-name, .store-name, [class*="seller-title"], [class*="merchant-name"]')?.textContent.trim() || null;
|
||||
if (!result.description) result.description = document.querySelector('.seller-description, .about-seller, [class*="seller-bio"], [class*="store-description"]')?.textContent.trim() || null;
|
||||
if (!result.rating) {
|
||||
const rEl = document.querySelector('[class*="rating"] [class*="value"], [class*="feedback-score"]');
|
||||
if (rEl) result.rating = parseFloat(rEl.textContent.replace(/[^0-9.]/g, '')) || null;
|
||||
}
|
||||
|
||||
const joinedEl = document.querySelector('[class*="joined"], [class*="member-since"], [class*="since"]');
|
||||
if (joinedEl) result.joined = joinedEl.textContent.trim();
|
||||
|
||||
const returnEl = document.querySelector('[class*="return-policy"] p, [class*="returns-policy"]');
|
||||
if (returnEl) result.return_policy = returnEl.textContent.trim().slice(0, 300);
|
||||
|
||||
result.url = window.location.href;
|
||||
|
||||
if (!result.name) {
|
||||
return JSON.stringify({ error: true, message: 'No seller data found. Ensure this is a seller or merchant profile page.' });
|
||||
}
|
||||
return JSON.stringify(result);
|
||||
} catch(e) {
|
||||
return JSON.stringify({ error: true, message: e.message });
|
||||
}
|
||||
})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,120 @@
|
||||
---
|
||||
name: goofish-item-detail
|
||||
description: "Extracts full detail data from a single Goofish (闲鱼/xianyu, goofish.com) second-hand item page. Input: item URL or item ID. Output: title, price, seller info (name, labels), full description, image gallery, item tags/attributes, want-count. Use when user mentions goofish item detail, 闲鱼商品详情, xianyu item page, 二手商品详情, get goofish product info, 采集闲鱼单品数据, 抓取闲鱼商品, scrape goofish item, xianyu product detail, 获取闲鱼卖家信息, seller info goofish, item description goofish, 闲鱼详情页, 想要人数, 闲鱼图片. Also applies to: verifying a specific listing before purchase, extracting seller contact/rating information, bulk item detail enrichment from a list of item IDs."
|
||||
---
|
||||
|
||||
# Goofish (闲鱼) — Item Detail
|
||||
|
||||
> item URL (or item_id + category_id) → full listing data: title, price, seller info, description, images, tags, want-count
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Load a single Goofish item detail page and extract its complete listing data including seller information, item description, image gallery, and attribute tags.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Browser with an active Goofish session (login required — item detail pages require authenticated access)
|
||||
- Target item URL format: `https://www.goofish.com/item?id={item_id}&categoryId={category_id}`
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
### 2. Login Verification
|
||||
|
||||
If login status for Goofish has been confirmed in the current session → skip this step.
|
||||
|
||||
Otherwise: open `https://www.goofish.com/` and observe the page:
|
||||
- User avatar or account entry exists → logged in, continue
|
||||
- Login/register prompt → not logged in; inform user that login is required; assist login flow
|
||||
|
||||
User refuses or cannot log in → terminate execution.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed on the page. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. `$(...)` is bash syntax; use the bash tool for execution.
|
||||
|
||||
### Network Capture: load item detail page
|
||||
|
||||
The item detail API (`mtop.taobao.idle.pc.detail/1.0/`) auto-fires when navigating to the item URL. Provide parameters via URL:
|
||||
|
||||
1. `navigate https://www.goofish.com/item?id={item_id}&categoryId={category_id}`
|
||||
2. `wait stable`
|
||||
3. Proceed to DOM extraction below
|
||||
|
||||
Error handling:
|
||||
- If a CAPTCHA slider appears ("Please slide to verify"): the session is rate-limited. Wait 5–10 minutes, or use remote-assist to complete the slider manually, then re-navigate.
|
||||
- If "网络不见了" error page appears: the item page API call failed. Retry once after 30 seconds; if it persists, the session may be temporarily blocked.
|
||||
- If item shows "该宝贝已下架" or similar: item has been removed/sold — skip and move to next item.
|
||||
|
||||
Note: Navigating to item detail pages in rapid succession (e.g., less than 2 seconds apart) increases the probability of CAPTCHA. Add 2–5 second delays between items in batch mode.
|
||||
|
||||
### DOM: extract item detail data
|
||||
|
||||
After navigating and waiting stable, extract all available fields:
|
||||
|
||||
`eval "$(python scripts/extract-item-detail.py)"`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"item_id": "1054899470781",
|
||||
"item_url": "https://www.goofish.com/item?id=1054899470781&categoryId=126862528",
|
||||
"title": "出一台iPhone 15 128G，电池健康度高，成色九新以上...",
|
||||
"price": "898.00",
|
||||
"original_price": "899.00",
|
||||
"seller_name": "小王数码严选",
|
||||
"seller_avatar": "https://img.alicdn.com/bao/uploaded/...",
|
||||
"seller_labels": ["成都", "刚刚擦亮", "来闲鱼5年", "卖出204件宝贝", "好评率100%"],
|
||||
"description": "出一台iPhone 15 128G，电池健康度高，成色九新以上...",
|
||||
"images": [
|
||||
"https://img.alicdn.com/bao/uploaded/i4/...",
|
||||
"https://img.alicdn.com/bao/uploaded/i2/..."
|
||||
],
|
||||
"tags": ["品牌：Apple/苹果", "型号：iPhone 15", "存储容量：128GB", "运行内存：6GB", "成色：几乎全新"],
|
||||
"want_count": "8人想要"
|
||||
}
|
||||
```
|
||||
|
||||
Fields that may be null: `original_price`, `seller_labels`, `description`, `tags`, `want_count` (depending on listing completeness). Note: on Goofish detail pages, `title` and `description` contain the same text — there is no separate title element.
|
||||
|
||||
## Pagination
|
||||
|
||||
N/A — single item page, no pagination.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`item_id non-null` and `title non-null` and `price non-null`
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Seller user ID (numeric) is not exposed in the DOM — only seller display name is available
|
||||
- Item detail pages trigger CAPTCHA if accessed too rapidly; recommended minimum interval: 3 seconds between items
|
||||
- Some items may require login even for viewing; the Skill will return an error if the page fails to load
|
||||
- Rapid successive navigation may trigger "网络不见了" error — wait 30 seconds before retrying
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Loop through item IDs serially with 3–5 second delays; do not parallelize within one browser. To increase throughput, distribute items across multiple browser sessions
|
||||
- **Test before batch execution**: Test with 2–3 items first to confirm selectors are valid; only then run the full batch
|
||||
- **Error resumption**: Save results item-by-item; on failure (CAPTCHA or error), record the failed item ID and resume after the CAPTCHA clears
|
||||
- **Get item IDs from search**: Use `goofish-search-list` Skill to collect item IDs, then feed them into this Skill for detail enrichment
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/xianyu-scraper-goofish-item-detail.memory.md`
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file.
|
||||
@@ -0,0 +1,118 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
args = parser.parse_args()
|
||||
|
||||
js = r"""
|
||||
(function() {
|
||||
try {
|
||||
var bodyText = document.body.innerText || '';
|
||||
if (bodyText.includes('网络不见了') || bodyText.includes('页面不存在') || bodyText.includes('糟糕！宝贝被删掉了')) {
|
||||
return JSON.stringify({ error: true, message: 'Page failed to load or item removed' });
|
||||
}
|
||||
|
||||
var captcha = document.querySelector('[class*="baxia-dialog"], [class*="punish"]');
|
||||
if (captcha && captcha.offsetParent !== null) {
|
||||
return JSON.stringify({ error: true, message: 'CAPTCHA verification required' });
|
||||
}
|
||||
|
||||
var itemContainer = document.querySelector('[class*="item-container"]');
|
||||
if (!itemContainer) {
|
||||
return JSON.stringify({ error: true, message: 'Item container not found — page may still be loading' });
|
||||
}
|
||||
|
||||
// Description (also serves as title on Goofish detail pages)
|
||||
var descEl = document.querySelector('[class*="main--"][class*="open"], span[class*="desc--"]');
|
||||
var description = descEl ? descEl.textContent.trim() : null;
|
||||
var title = description;
|
||||
|
||||
// Price: current/bargain price in item-main-info area
|
||||
var priceEl = document.querySelector('[class*="item-main-info"] [class*="price--"][class*="windows"]');
|
||||
if (!priceEl) {
|
||||
priceEl = document.querySelector('[class*="value--"] [class*="price--"]');
|
||||
}
|
||||
var price = priceEl ? priceEl.textContent.trim() : null;
|
||||
|
||||
// Original price from tips area
|
||||
var tipsEl = document.querySelector('[class*="item-main-info"] [class*="tips--"]');
|
||||
var originalPrice = null;
|
||||
if (tipsEl) {
|
||||
var tipsText = tipsEl.textContent || '';
|
||||
var opMatch = tipsText.match(/直接买\\s*[￥¥](\\d+(?:\\.\\d{1,2})?)/);
|
||||
if (opMatch) originalPrice = opMatch[1];
|
||||
}
|
||||
|
||||
// Seller info
|
||||
var nickEl = document.querySelector('[class*="item-user-info-nick"]');
|
||||
var sellerName = nickEl ? nickEl.textContent.trim() : null;
|
||||
|
||||
var avatarEl = document.querySelector('[class*="item-user-info-avatar"] img');
|
||||
var sellerAvatar = avatarEl ? (avatarEl.src || null) : null;
|
||||
|
||||
var labelEls = document.querySelectorAll('[class*="item-user-info-label"]');
|
||||
var sellerLabels = Array.from(labelEls).map(function(el) { return el.textContent.trim(); }).filter(Boolean);
|
||||
|
||||
// Images from gallery (deduplicate by base filename)
|
||||
var imgEls = document.querySelectorAll('[class*="item-main-window"] img[src*="alicdn"]');
|
||||
if (imgEls.length === 0) {
|
||||
imgEls = document.querySelectorAll('[class*="item-main"] img[src*="alicdn"]');
|
||||
}
|
||||
var seen = {};
|
||||
var images = [];
|
||||
Array.from(imgEls).forEach(function(el) {
|
||||
var src = el.src || el.getAttribute('data-src');
|
||||
if (!src || src.indexOf('alicdn') === -1 || src.indexOf('tps-2-2') > -1) return;
|
||||
var keyMatch = src.match(/O1CN[^.]+/);
|
||||
var key = keyMatch ? keyMatch[0] : src;
|
||||
if (!seen[key]) {
|
||||
seen[key] = true;
|
||||
images.push(src);
|
||||
}
|
||||
});
|
||||
|
||||
// Item attributes/tags
|
||||
var tagEls = document.querySelectorAll('[class*="labels--"] [class*="item--"]');
|
||||
var tags = Array.from(tagEls).map(function(el) { return el.textContent.trim(); }).filter(Boolean);
|
||||
|
||||
// Want count
|
||||
var wantEl = document.querySelector('[class*="want--"]');
|
||||
var wantCount = null;
|
||||
if (wantEl) {
|
||||
var wantText = wantEl.textContent || '';
|
||||
var wantMatch = wantText.match(/(\\d+人想要)/);
|
||||
wantCount = wantMatch ? wantMatch[1] : wantText.trim();
|
||||
}
|
||||
|
||||
var itemUrl = window.location.href;
|
||||
var idMatch = itemUrl.match(/[?&]id=(\\d+)/);
|
||||
|
||||
if (!description && !price) {
|
||||
return JSON.stringify({ error: true, message: 'Item content not loaded — page may still be loading' });
|
||||
}
|
||||
|
||||
return JSON.stringify({
|
||||
item_id: idMatch ? idMatch[1] : null,
|
||||
item_url: itemUrl,
|
||||
title: title,
|
||||
price: price,
|
||||
original_price: originalPrice,
|
||||
seller_name: sellerName,
|
||||
seller_avatar: sellerAvatar,
|
||||
seller_labels: sellerLabels.length > 0 ? sellerLabels : null,
|
||||
description: description,
|
||||
images: images.length > 0 ? images : null,
|
||||
tags: tags.length > 0 ? tags : null,
|
||||
want_count: wantCount
|
||||
});
|
||||
} catch(e) {
|
||||
return JSON.stringify({ error: true, message: e.message });
|
||||
}
|
||||
})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,156 @@
|
||||
---
|
||||
name: goofish-search-list
|
||||
description: "Scrapes second-hand item search results from Goofish (闲鱼/xianyu, goofish.com) — China's largest second-hand marketplace. Input: keyword, optional sort/filter params. Output: list of items with id, title, price, image, location, want-count per page (30 items/page). Use when user mentions goofish, 闲鱼, xianyu, 二手交易, second-hand marketplace China, 二手商品搜索, search used goods, scrape goofish listings, xianyu search results, collect second-hand prices, monitor used item prices, 闲鱼关键词搜索, 闲鱼数据采集, 批量抓取闲鱼, goofish scraper, goofish data, xianyu data extraction, 二手商品价格监控, used iPhone prices, 二手手机价格. Also applies to: price research on Chinese second-hand market, competitor product monitoring via used goods listings, inventory analysis."
|
||||
---
|
||||
|
||||
# Goofish (闲鱼) — Search Results List
|
||||
|
||||
> keyword + optional filters → list of 30 second-hand item cards per page (id, title, price, image, location, want-count)
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Extract second-hand item listing cards from Goofish keyword search results, supporting sort options, price range filters, and publish-date filters, with page-by-page pagination.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Browser with an active Goofish session (logged-in account recommended for full results)
|
||||
- Target page is already open or will be opened: `https://www.goofish.com/search?q={keyword}`
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
### 2. Login Verification
|
||||
|
||||
If login status for Goofish has been confirmed in the current session → skip this step.
|
||||
|
||||
Otherwise: open `https://www.goofish.com/` and observe the page:
|
||||
- User avatar or account entry exists → logged in, continue
|
||||
- Login/register prompt → not logged in; inform user that login may be required for full results; assist login if needed
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. `$(...)` is bash syntax; it is recommended to use the bash tool for execution.
|
||||
|
||||
### Network Capture: trigger search and load results
|
||||
|
||||
Search requests use a dynamic `sign` token computed client-side — they cannot be reconstructed directly. Navigate to the search URL to trigger the API automatically.
|
||||
|
||||
1. `navigate https://www.goofish.com/search?q={keyword}`
|
||||
2. `wait stable`
|
||||
3. Proceed to DOM extraction below
|
||||
|
||||
Error handling: If the page shows a CAPTCHA slider ("Please slide to verify") instead of search results, the session has been rate-limited. Wait 5–10 minutes before retrying, or switch to a fresh browser session.
|
||||
|
||||
### DOM: search result item cards (data extraction)
|
||||
|
||||
After navigating and waiting stable, extract all 30 item cards on the current page:
|
||||
|
||||
`eval "$(python scripts/extract-search-items.py)"`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"item_id": "1054668718340", // unique item ID
|
||||
"category_id": "126862528", // category ID
|
||||
"item_url": "https://www.goofish.com/item?id=1054668718340&categoryId=126862528",
|
||||
"title": "美版iPhone 14 国行256G 纯原 原版原漆", // full title text
|
||||
"image_url": "https://img.alicdn.com/bao/uploaded/...", // thumbnail URL
|
||||
"price": "1810", // numeric string, CNY, no ¥ sign
|
||||
"service_tag": "Apple/苹果256GB无任何维修", // condition/attribute tag or recency label, null if absent
|
||||
"price_desc": "2人想要", // want-count or price-drop info, null if absent
|
||||
"location": "广东" // seller's location province/city
|
||||
}
|
||||
],
|
||||
"count": 30
|
||||
}
|
||||
```
|
||||
|
||||
### DOM: apply sort and filter options (operation)
|
||||
|
||||
Apply sort order, publish-date filter, or price range before extracting. Call before running `extract-search-items.py`. After calling, `wait stable` before extracting.
|
||||
|
||||
`eval "$(python scripts/apply-search-filters.py --sort {sort} --publish-days {days} --price-min {min} --price-max {max})"`
|
||||
|
||||
Parameters:
|
||||
- `--sort`: Sort option — `""` default (综合), `"reduce"` price-drop (新降价), `"create"` newest (新发布), `"price-asc"` price low-to-high, `"price-desc"` price high-to-low. Default: `""`
|
||||
- `--publish-days`: Filter by publish date — `""` all, `"1"` within 1 day, `"3"` within 3 days, `"7"` within 7 days, `"14"` within 14 days. Default: `""`
|
||||
- `--price-min`: Minimum price (CNY integer string, e.g., `"500"`). Requires `--price-max`. Default: `""`
|
||||
- `--price-max`: Maximum price (CNY integer string, e.g., `"3000"`). Requires `--price-min`. Default: `""`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"ok": true,
|
||||
"applied": {
|
||||
"sort": "reduce:desc",
|
||||
"searchFilter": "publishDays:7;priceRange:500,3000;"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### DOM: navigate to a specific page (operation)
|
||||
|
||||
`eval "$(python scripts/goto-page.py {page_number})"`
|
||||
|
||||
Parameters:
|
||||
- `page_number`: Target page number (integer, 1-based)
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{ "ok": true, "clicked_page": 2 }
|
||||
```
|
||||
|
||||
After clicking, `wait stable` then re-run `extract-search-items.py` to get the new page's items.
|
||||
|
||||
## Enum Parameters
|
||||
|
||||
[AI] sort options: `""` (综合/default), `"reduce"` (新降价), `"create"` (新发布/最新), `"price-asc"` (价格从低到高), `"price-desc"` (价格从高到低)
|
||||
|
||||
[AI] publish-days filter: `""` (all), `"1"`, `"3"`, `"7"`, `"14"`
|
||||
|
||||
## Pagination
|
||||
|
||||
**DOM Pagination**: Click the target page number button using `goto-page.py {page}`, then `wait stable`, then re-run `extract-search-items.py`. Page numbers appear in the pagination bar at the bottom of the search results.
|
||||
|
||||
Termination: When `goto-page.py` returns `error: Page N not found` — no more pages available, or the target page exceeds the pagination range displayed (typically up to 25 pages / 750 items).
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`result count >= 1` and `item_id non-null rate = 100%` and `price non-null rate >= 80%`
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- 30 items per page (fixed by the site)
|
||||
- Maximum ~750 items accessible via pagination (25 pages × 30)
|
||||
- Seller username and user ID are not available in search cards — only seller location
|
||||
- Session rate limiting: accessing item detail pages rapidly after heavy search usage may trigger a CAPTCHA slider; mitigate by adding 1–2 second delays between page navigations
|
||||
- The `sign` token in search API requests is computed client-side; direct API replay without browser context is not supported — always trigger via page navigation
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script to loop through keywords serially within a single session; do not parallelize within one browser (prone to triggering anti-scraping). Add 1–2 second delays between page navigations. To increase throughput, open multiple stealth browser sessions and distribute keywords across them
|
||||
- **Test before batch execution**: After writing a batch script, first test with 1–2 keywords/pages to verify the script runs correctly; only then run the full batch
|
||||
- **Reduce redundant pre-operations**: Navigate once per keyword, apply all filters at once before extracting, rather than navigating multiple times
|
||||
- **Error resumption**: Save results keyword-by-keyword and page-by-page during batch processing; on failure, resume from the last saved position
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/xianyu-scraper-goofish-search-list.memory.md`
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file. Do not record what keywords were used or how many results were returned — those are task outputs, not experience.
|
||||
@@ -0,0 +1,121 @@
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--sort', default='',
|
||||
help='Sort option: "" (default), "reduce" (price-drop), "create" (newest), "price-asc", "price-desc"')
|
||||
parser.add_argument('--publish-days', default='',
|
||||
help='Filter by publish date: "" (all), "1", "3", "7", "14"')
|
||||
parser.add_argument('--price-min', default='', help='Min price filter (e.g., 500)')
|
||||
parser.add_argument('--price-max', default='', help='Max price filter (e.g., 5000)')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Build searchFilter string
|
||||
search_filter_parts = []
|
||||
if args.publish_days:
|
||||
search_filter_parts.append(f'publishDays:{args.publish_days};')
|
||||
if args.price_min and args.price_max:
|
||||
search_filter_parts.append(f'priceRange:{args.price_min},{args.price_max};')
|
||||
search_filter = ''.join(search_filter_parts)
|
||||
|
||||
# Resolve sort field/value
|
||||
sort_map = {
|
||||
'': ('', ''),
|
||||
'reduce': ('reduce', 'desc'),
|
||||
'create': ('create', 'desc'),
|
||||
'price-asc': ('price', 'asc'),
|
||||
'price-desc': ('price', 'desc'),
|
||||
}
|
||||
sort_field, sort_value = sort_map.get(args.sort, ('', ''))
|
||||
|
||||
js = f"""
|
||||
(function() {{
|
||||
try {{
|
||||
var sortField = {json.dumps(sort_field)};
|
||||
var sortValue = {json.dumps(sort_value)};
|
||||
var searchFilter = {json.dumps(search_filter)};
|
||||
var priceMin = {json.dumps(args.price_min)};
|
||||
var priceMax = {json.dumps(args.price_max)};
|
||||
|
||||
// Apply sort if needed
|
||||
if (sortField === 'reduce') {{
|
||||
// Click 新降价 direct button
|
||||
var sortBtns = document.querySelectorAll('[class*="search-select-title"]');
|
||||
for (var i = 0; i < sortBtns.length; i++) {{
|
||||
if (sortBtns[i].textContent.trim() === '新降价') {{
|
||||
sortBtns[i].click();
|
||||
break;
|
||||
}}
|
||||
}}
|
||||
}} else if (sortField === 'create') {{
|
||||
// Click 最新 inside 新发布 dropdown
|
||||
var items = document.querySelectorAll('[class*="search-select-item"]');
|
||||
for (var i = 0; i < items.length; i++) {{
|
||||
if (items[i].textContent.trim() === '最新') {{
|
||||
items[i].click();
|
||||
break;
|
||||
}}
|
||||
}}
|
||||
}} else if (sortField === 'price' && sortValue === 'asc') {{
|
||||
var items = document.querySelectorAll('[class*="search-select-item"]');
|
||||
for (var i = 0; i < items.length; i++) {{
|
||||
if (items[i].textContent.trim() === '价格从低到高') {{
|
||||
items[i].click();
|
||||
break;
|
||||
}}
|
||||
}}
|
||||
}} else if (sortField === 'price' && sortValue === 'desc') {{
|
||||
var items = document.querySelectorAll('[class*="search-select-item"]');
|
||||
for (var i = 0; i < items.length; i++) {{
|
||||
if (items[i].textContent.trim() === '价格从高到低') {{
|
||||
items[i].click();
|
||||
break;
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
// Apply publishDays filter (inside 新发布 dropdown)
|
||||
if (searchFilter.indexOf('publishDays') >= 0) {{
|
||||
var dayMatch = searchFilter.match(/publishDays:(\\d+)/);
|
||||
if (dayMatch) {{
|
||||
var dayLabel = dayMatch[1] === '1' ? '1天内' : (dayMatch[1] + '天内');
|
||||
var items = document.querySelectorAll('[class*="search-select-item"]');
|
||||
for (var i = 0; i < items.length; i++) {{
|
||||
if (items[i].textContent.trim() === dayLabel) {{
|
||||
items[i].click();
|
||||
break;
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
// Apply price range filter
|
||||
if (priceMin && priceMax) {{
|
||||
var inputs = document.querySelectorAll('[class*="search-price-input"] input');
|
||||
if (inputs.length >= 2) {{
|
||||
var setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
|
||||
setter.call(inputs[0], priceMin);
|
||||
inputs[0].dispatchEvent(new Event('input', {{bubbles: true}}));
|
||||
setter.call(inputs[1], priceMax);
|
||||
inputs[1].dispatchEvent(new Event('input', {{bubbles: true}}));
|
||||
var confirmBtn = document.querySelector('[class*="search-price-confirm-button"]');
|
||||
if (confirmBtn) confirmBtn.click();
|
||||
}}
|
||||
}}
|
||||
|
||||
return JSON.stringify({{ ok: true, applied: {{
|
||||
sort: sortField + (sortValue ? ':' + sortValue : ''),
|
||||
searchFilter: searchFilter
|
||||
}} }});
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{ error: true, message: e.message }});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,62 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
args = parser.parse_args()
|
||||
|
||||
js = f"""
|
||||
(function() {{
|
||||
try {{
|
||||
var items = document.querySelectorAll('a[class*="feeds-item-wrap"]');
|
||||
if (items.length === 0) {{
|
||||
return JSON.stringify({{ error: true, message: 'No search result items found — page may not have loaded or search returned no results' }});
|
||||
}}
|
||||
var result = [];
|
||||
for (var i = 0; i < items.length; i++) {{
|
||||
var item = items[i];
|
||||
var href = item.getAttribute('href') || '';
|
||||
var idMatch = href.match(/[?&]id=(\\d+)/);
|
||||
var catMatch = href.match(/categoryId=(\\d+)/);
|
||||
|
||||
var titleEl = item.querySelector('[class*="row1-wrap-title"]');
|
||||
var title = titleEl ? (titleEl.getAttribute('title') || titleEl.textContent.replace(/^\\s+|\\s+$/g, '')) : null;
|
||||
|
||||
var imgEl = item.querySelector('img[class*="feeds-image"]');
|
||||
|
||||
var numEl = item.querySelector('[class*="number--"]');
|
||||
var decEl = item.querySelector('[class*="decimal--"]');
|
||||
var price = (numEl ? numEl.textContent.trim() : '') + (decEl ? decEl.textContent.trim() : '');
|
||||
|
||||
var tagEl = item.querySelector('[class*="row2-wrap"]');
|
||||
var serviceTag = tagEl ? tagEl.textContent.trim() : null;
|
||||
|
||||
var descEl = item.querySelector('[class*="price-desc"]');
|
||||
var priceDesc = descEl ? descEl.textContent.trim() : null;
|
||||
|
||||
var locEl = item.querySelector('[class*="seller-text--"]');
|
||||
var location = locEl ? locEl.textContent.trim() : null;
|
||||
|
||||
result.push({{
|
||||
item_id: idMatch ? idMatch[1] : null,
|
||||
category_id: catMatch ? catMatch[1] : null,
|
||||
item_url: href,
|
||||
title: title,
|
||||
image_url: imgEl ? imgEl.src : null,
|
||||
price: price || null,
|
||||
service_tag: serviceTag || null,
|
||||
price_desc: priceDesc || null,
|
||||
location: location
|
||||
}});
|
||||
}}
|
||||
return JSON.stringify({{ items: result, count: result.length }});
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{ error: true, message: e.message }});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,33 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('page', type=int, help='Target page number (1-based)')
|
||||
args = parser.parse_args()
|
||||
|
||||
js = f"""
|
||||
(function() {{
|
||||
try {{
|
||||
var targetPage = {args.page};
|
||||
var pages = document.querySelectorAll('[class*="search-pagination-page-box"]');
|
||||
if (pages.length === 0) {{
|
||||
return JSON.stringify({{ error: true, message: 'Pagination not found — results may be too few for multiple pages' }});
|
||||
}}
|
||||
for (var i = 0; i < pages.length; i++) {{
|
||||
if (parseInt(pages[i].textContent.trim()) === targetPage) {{
|
||||
pages[i].click();
|
||||
return JSON.stringify({{ ok: true, clicked_page: targetPage }});
|
||||
}}
|
||||
}}
|
||||
return JSON.stringify({{ error: true, message: 'Page ' + targetPage + ' not found in pagination. Available pages: ' + Array.from(pages).map(function(p) {{ return p.textContent.trim(); }}).join(', ') }});
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{ error: true, message: e.message }});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,122 @@
|
||||
---
|
||||
name: taobao-keyword-search
|
||||
description: "Search Taobao and Tmall product listings by keyword, returning paginated product cards with title, price, shop, image, sales, and tags. Use when user asks to search Taobao, find products on Taobao/Tmall, scrape Taobao search results, get product listings from Taobao, collect Taobao items by keyword, 搜索淘宝, 淘宝关键词搜索, 采集淘宝商品, 抓取淘宝搜索结果, 淘宝天猫商品列表. Also applies to bulk keyword searches, price monitoring across keywords, and competitive product research on Taobao."
|
||||
---
|
||||
|
||||
# Taobao — Keyword Search
|
||||
|
||||
> keyword + optional filters → paginated product listing (itemId, title, price, shop, image, sales)
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Search Taobao/Tmall for products by keyword and extract product cards from search results pages.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Target page is already open in the browser: `https://s.taobao.com/search`
|
||||
- User is logged in to Taobao (user avatar or nickname visible in the page header)
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
### 2. Login Verification
|
||||
|
||||
If login status for Taobao has been confirmed in the current session → skip this step.
|
||||
|
||||
Otherwise: open `https://www.taobao.com` and observe the page header:
|
||||
- User nickname visible (e.g., "心林vs妞妞") → logged in, continue execution
|
||||
- "亲，请登录" or login button visible → not logged in, inform the user that Taobao login is needed first, assist the user in completing the login flow
|
||||
|
||||
User refuses or cannot log in → terminate execution.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page, never bypassing authentication or access controls. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. `$(...)` is bash syntax; it is recommended to use the bash tool for execution.
|
||||
|
||||
### DOM: product search results (data extraction)
|
||||
|
||||
Navigate to search results URL, then extract:
|
||||
|
||||
1. `navigate "https://s.taobao.com/search?q={keyword}&page={page}&ie=utf8"`
|
||||
2. `wait stable`
|
||||
3. `eval "$(python scripts/search-products.py '{keyword}' --page {page} --sort '{sort}' --tab '{tab}' --start-price {startPrice} --end-price {endPrice})"`
|
||||
|
||||
URL Parameters:
|
||||
- `q`: URL-encoded keyword
|
||||
- `page`: page number, 1-based, default `1`
|
||||
- `sort`: sort order — empty string (default/recommended), `sale-desc` (by sales), `price-asc` (price low→high), `price-desc` (price high→low)
|
||||
- `tab`: `mall` for Tmall-only results; omit for all results
|
||||
- `startPrice` / `endPrice`: price range filter in yuan (e.g., `startPrice=100&endPrice=500`)
|
||||
|
||||
Output example:
|
||||
```json
|
||||
[
|
||||
{
|
||||
"itemId": "694593508978",
|
||||
"itemUrl": "https://item.taobao.com/item.htm?id=694593508978",
|
||||
"title": "蓝牙耳机2025新款官方",
|
||||
"subTitle": "AI耳机热卖榜第1名",
|
||||
"priceYuan": 79.9,
|
||||
"priceDesc": "券后价",
|
||||
"imageUrl": "https://img.alicdn.com/imgextra/...",
|
||||
"salesCount": "40万+人付款",
|
||||
"shopName": "金运旗舰店",
|
||||
"location": "广东",
|
||||
"rating": null,
|
||||
"tags": ["政府补贴15%", "官方立减26元"]
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Notes:
|
||||
- `priceYuan` is the displayed price (may be post-coupon price, not pre-coupon)
|
||||
- `priceDesc` indicates price type: `券后价` (after coupon), `补贴价` (subsidized price), etc.
|
||||
- `rating` is rarely shown on search cards; null is expected
|
||||
- Sponsored/ad items have `itemUrl` pointing to `click.simba.taobao.com` — they will have `itemId` extracted from query params
|
||||
|
||||
## Enum Parameters
|
||||
|
||||
[collection failed] `sort` values: confirmed values are empty string (default), `sale-desc`, `price-asc`, `price-desc`; no API for enumeration, values are hardcoded constants.
|
||||
|
||||
## Pagination
|
||||
|
||||
**URL Pagination**: URL pattern `https://s.taobao.com/search?q={keyword}&page={N}&ie=utf8`, increment `page` from 1. Each page returns ~47 items. Termination: when `result count < 10` or returned items duplicate previous page.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`result count >= 1` and `itemId` non-null rate = 100%
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Requires Taobao login; unauthenticated sessions redirect to login page
|
||||
- Prices shown are displayed prices (may be post-coupon), not pre-discount prices
|
||||
- Sponsored/ad items appear at unpredictable positions in results
|
||||
- Price filter (`startPrice`/`endPrice`) filters on pre-coupon prices, so post-coupon prices displayed may fall outside the requested range near boundaries
|
||||
- Taobao may return different result counts across pages; page count is approximate
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script to loop through keywords serially within a single session; do not parallelize within one browser (prone to triggering anti-scraping restrictions). Add 2–3 second intervals between page navigations. To increase throughput, open multiple stealth browser sessions and distribute keywords across them.
|
||||
- **Test before batch execution**: After writing a batch script, you must first test with 1–2 keywords to verify the script runs correctly; only then run the full batch. Never skip testing and execute in batch directly.
|
||||
- **Reduce redundant pre-operations**: When scraping multiple pages for one keyword, navigate page 2, 3 etc. within the same session without re-login checks.
|
||||
- **Error resumption**: Save results item by item during batch processing; on failure, resume from the breakpoint rather than starting over.
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/taobao-keyword-search.memory.md`
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file. Do not record what keywords were used or how many results were returned — those are task outputs, not experience.
|
||||
@@ -0,0 +1,67 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('keyword') # search keyword (for documentation only; URL already set)
|
||||
parser.add_argument('--page', default='1') # page number, 1-based
|
||||
parser.add_argument('--sort', default='') # sort: '', 'sale-desc', 'price-asc', 'price-desc'
|
||||
parser.add_argument('--tab', default='') # 'mall' for Tmall-only, '' for all
|
||||
parser.add_argument('--start-price', default='') # min price in yuan
|
||||
parser.add_argument('--end-price', default='') # max price in yuan
|
||||
args = parser.parse_args()
|
||||
|
||||
js = f"""
|
||||
(function() {{
|
||||
try {{
|
||||
var cards = document.querySelectorAll('[id^="item_id_"]');
|
||||
if (!cards || cards.length === 0) {{
|
||||
return JSON.stringify({{ error: true, message: 'No product cards found. Page may not have loaded or login is required.' }});
|
||||
}}
|
||||
var items = Array.from(cards).map(function(card) {{
|
||||
var a = card.tagName === 'A' ? card : card.querySelector('a[href*="item.taobao.com"], a[href*="detail.tmall.com"]');
|
||||
var href = a ? a.href : '';
|
||||
var url;
|
||||
try {{ url = new URL(href || 'https://taobao.com'); }} catch(e) {{ url = new URL('https://taobao.com'); }}
|
||||
var img = card.querySelector('img[class*="mainImg"]') || card.querySelector('img');
|
||||
var priceInt = card.querySelector('[class*="priceInt--"]');
|
||||
var priceFloat = card.querySelector('[class*="priceFloat--"]');
|
||||
var priceDesc = card.querySelector('[class*="priceDesc--"]');
|
||||
var sales = card.querySelector('[class*="realSales--"]');
|
||||
var shopName = card.querySelector('[class*="shopNameText--"]');
|
||||
var location = card.querySelector('[class*="provcity--"], [class*="location--"]');
|
||||
var titleEl = card.querySelector('[class*="title--"], [class*="itemTitle--"]');
|
||||
var subTitle = card.querySelector('[class*="subTitle--"]');
|
||||
var rate = card.querySelector('[class*="shopRating--"], [class*="rating--"]');
|
||||
var tags = Array.from(card.querySelectorAll('[class*="tag--"], [class*="Tag--"], [class*="label--"]'))
|
||||
.map(function(t) {{ return t.textContent.trim(); }})
|
||||
.filter(function(t) {{ return t.length > 0; }})
|
||||
.slice(0, 5);
|
||||
var rawId = url.searchParams.get('id') || card.id.replace('item_id_', '');
|
||||
var rawHref = href.indexOf('simba.taobao.com') >= 0 ? href : (href.split('?')[0] + (rawId ? '?id=' + rawId : ''));
|
||||
return {{
|
||||
itemId: rawId,
|
||||
itemUrl: rawHref,
|
||||
title: (titleEl ? titleEl.textContent.trim() : (img ? img.alt : '')) || '',
|
||||
subTitle: subTitle ? subTitle.textContent.trim() : '',
|
||||
priceYuan: (priceInt && priceFloat) ? parseFloat(priceInt.textContent.trim() + priceFloat.textContent.trim()) : null,
|
||||
priceDesc: priceDesc ? priceDesc.textContent.trim() : '',
|
||||
imageUrl: img ? img.src : '',
|
||||
salesCount: sales ? sales.textContent.trim() : '',
|
||||
shopName: shopName ? shopName.textContent.trim() : '',
|
||||
location: location ? location.textContent.trim() : '',
|
||||
rating: rate ? rate.textContent.trim() : null,
|
||||
tags: tags
|
||||
}};
|
||||
}});
|
||||
return JSON.stringify(items);
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{ error: true, message: e.message }});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,127 @@
|
||||
---
|
||||
name: taobao-product-detail
|
||||
description: "Fetch full product detail from a Taobao or Tmall product page by itemId, returning title, price, shop info, images, SKU variants, and product attributes. Use when user asks to get product details from Taobao, scrape a Taobao item page, extract product info by item ID, fetch Tmall product data, 抓取淘宝商品详情, 获取淘宝商品信息, 淘宝商品页面采集, 天猫商品详情, 按商品ID获取信息. Also applies to building product databases, price tracking by itemId, and product comparison research."
|
||||
---
|
||||
|
||||
# Taobao — Product Detail
|
||||
|
||||
> itemId → full product detail (title, price, shop, images, SKU variants, attributes)
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Navigate to a Taobao or Tmall product page and extract full product information from the DOM.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Target page is already open in the browser: `https://item.taobao.com/item.htm?id={itemId}`
|
||||
- User is logged in to Taobao (user avatar or nickname visible in the page header)
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
### 2. Login Verification
|
||||
|
||||
If login status for Taobao has been confirmed in the current session → skip this step.
|
||||
|
||||
Otherwise: open `https://www.taobao.com` and observe the page header:
|
||||
- User nickname visible → logged in, continue execution
|
||||
- Login button visible → not logged in, inform the user that Taobao login is needed first, assist the user in completing the login flow
|
||||
|
||||
User refuses or cannot log in → terminate execution.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page, never bypassing authentication or access controls. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. `$(...)` is bash syntax; it is recommended to use the bash tool for execution.
|
||||
|
||||
### DOM: product detail page (data extraction)
|
||||
|
||||
Navigate to the product page and extract all fields:
|
||||
|
||||
1. `navigate "https://item.taobao.com/item.htm?id={itemId}"`
|
||||
2. `wait stable`
|
||||
3. Close any popup if present: look for buttons with text "开心收下", "不了", "关闭" and click to dismiss
|
||||
4. `eval "$(python scripts/extract-product.py '{itemId}')"`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"itemId": "744983869996",
|
||||
"itemUrl": "https://detail.tmall.com/item.htm?id=744983869996",
|
||||
"isTmall": true,
|
||||
"title": "绿联转换插头英标马来西亚新加坡澳洲韩国新西兰Switch插头转换器",
|
||||
"price": 17.9,
|
||||
"priceFormatted": "￥17.9",
|
||||
"originalPrice": null,
|
||||
"shopName": "绿联数码旗舰店",
|
||||
"shopUrl": "https://shop67095450.taobao.com/category.htm",
|
||||
"shopId": "67095450",
|
||||
"images": [
|
||||
"https://img.alicdn.com/imgextra/i3/713464357/O1CN01fQN7GG1i3YdCfBjzF_!!0-item_pic.jpg"
|
||||
],
|
||||
"skuVariants": [
|
||||
"磨砂黑|英转中转换器【适用国内电器】适用马来西亚/新加坡等国家",
|
||||
"轻巧白|英转中转换器【适用国内电器】适用马来西亚/新加坡等国家"
|
||||
],
|
||||
"attributes": {
|
||||
"产地": "中国大陆",
|
||||
"品牌": "绿联",
|
||||
"转换器类型": "英标",
|
||||
"型号": "S510"
|
||||
},
|
||||
"reviewCount": "7000+"
|
||||
}
|
||||
```
|
||||
|
||||
Notes:
|
||||
- `isTmall`: true when product is on Tmall (URL contains `tmall.com`)
|
||||
- `price`: the currently displayed price (may be flash sale, subsidized, or post-coupon price); multiply SKU variants affect the displayed price
|
||||
- `originalPrice`: the crossed-out original price when a sale is active; null when no sale
|
||||
- `shopName`: cleaned shop name (shop header link text)
|
||||
- `images`: deduplicated, `.webp` suffix removed for cleaner URLs; first image is the main listing image
|
||||
- `skuVariants`: all visible SKU option labels (color, size, etc.)
|
||||
- `attributes`: key-value pairs from the product specifications section; may include `颜色分类` with all variant names as a combined string
|
||||
- `reviewCount`: approximate text from page (e.g., "7000+"), not a precise integer
|
||||
|
||||
Error handling: if `title` is null, the product page may not have loaded correctly — check if still on the product page (`state` to inspect URL) and retry navigation.
|
||||
|
||||
## Pagination
|
||||
|
||||
N/A — single product page, no pagination.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`title` non-null AND `itemId` matches input
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- `price` is the currently displayed price for the default/first SKU; to get prices for other SKU variants, click each `skuVariants` option and re-run the price extraction
|
||||
- Shop name in raw DOM concatenates rating text; the script extracts only the link text but may still include ratings in some layouts
|
||||
- Images include both product listing images and some thumbnail duplicates; the script deduplicates by URL
|
||||
- `originalPrice` extraction depends on the `subPrice--` class structure which varies by product type (flash sale vs regular discount)
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script to loop through itemIds serially within a single session; add 2–3 second intervals between navigations.
|
||||
- **Test before batch execution**: After writing a batch script, you must first test with 1–2 items to verify the script runs correctly; only then run the full batch. Never skip testing and execute in batch directly.
|
||||
- **Reduce redundant pre-operations**: When scraping multiple products, stay in the same session without re-login checks between items.
|
||||
- **Error resumption**: Save results item by item during batch processing; on failure, resume from the breakpoint rather than starting over.
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/taobao-product-detail.memory.md`
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file. Do not record what keywords were used or how many results were returned — those are task outputs, not experience.
|
||||
@@ -0,0 +1,84 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('item_id') # Taobao/Tmall itemId (for documentation only; page already loaded)
|
||||
args = parser.parse_args()
|
||||
|
||||
js = f"""
|
||||
(function() {{
|
||||
try {{
|
||||
var result = {{}};
|
||||
result.itemId = new URLSearchParams(location.search).get('id') || '{args.item_id}';
|
||||
result.itemUrl = location.href;
|
||||
result.isTmall = location.hostname.indexOf('tmall.com') >= 0;
|
||||
|
||||
var titleEl = document.querySelector('[class*="mainTitle--"]');
|
||||
result.title = titleEl ? titleEl.textContent.trim() : null;
|
||||
if (!result.title) {{
|
||||
return JSON.stringify({{ error: true, message: 'Title not found. Product page may not have loaded correctly.' }});
|
||||
}}
|
||||
|
||||
var priceWrap = document.querySelector('[class*="priceWrap--"]');
|
||||
var priceText = priceWrap ? priceWrap.querySelector('[class*="text--"]') : null;
|
||||
var priceSymbol = priceWrap ? priceWrap.querySelector('[class*="symbol--"]') : null;
|
||||
result.price = priceText ? parseFloat(priceText.textContent.trim()) : null;
|
||||
result.priceFormatted = (priceSymbol && priceText) ? (priceSymbol.textContent.trim() + priceText.textContent.trim()) : null;
|
||||
var subPriceText = priceWrap ? priceWrap.querySelector('[class*="subPrice--"] [class*="text--"]') : null;
|
||||
result.originalPrice = subPriceText ? parseFloat(subPriceText.textContent.trim()) : null;
|
||||
|
||||
var shopHeader = document.querySelector('[class*="shopHeader--"]');
|
||||
var shopLink = shopHeader ? shopHeader.querySelector('a[href*="taobao.com"]') : null;
|
||||
var shopLinkText = shopLink ? shopLink.textContent.trim() : '';
|
||||
// Extract just the shop name: remove rating/metric text that follows the name
|
||||
var shopNameMatch = shopLinkText.match(/^([\\u4e00-\\u9fa5a-zA-Z0-9 ·&()\\-_]+?)(?=[0-9]{{1}}\\.[0-9]|\\d{{2}}VIP|好评率|$)/);
|
||||
result.shopName = shopNameMatch ? shopNameMatch[1].trim() : shopLinkText.slice(0, 30).trim();
|
||||
result.shopUrl = shopLink ? shopLink.href : null;
|
||||
var shopIdMatch = (shopLink ? shopLink.href : '').match(/shop(\\d+)/);
|
||||
result.shopId = shopIdMatch ? shopIdMatch[1] : null;
|
||||
|
||||
var galleryImgs = Array.from(document.querySelectorAll('[class*="Gallery"] img'));
|
||||
var imgUrls = galleryImgs.map(function(img) {{
|
||||
return img.src.replace(/_q50[^.]*\\.jpg[^.]*$/, '').replace(/\\.webp$/, '');
|
||||
}}).filter(function(u) {{ return u && u.indexOf('alicdn.com') >= 0; }});
|
||||
var seenUrls = {{}};
|
||||
result.images = imgUrls.filter(function(u) {{
|
||||
if (seenUrls[u]) return false;
|
||||
seenUrls[u] = true;
|
||||
return true;
|
||||
}}).slice(0, 10);
|
||||
|
||||
var skuItems = Array.from(document.querySelectorAll('[class*="valueItem--"]'));
|
||||
result.skuVariants = skuItems.map(function(el) {{ return el.textContent.trim(); }});
|
||||
|
||||
var attrs = {{}};
|
||||
var emphTitles = Array.from(document.querySelectorAll('[class*="emphasisParamsInfoItemTitle--"]'));
|
||||
var emphSubs = Array.from(document.querySelectorAll('[class*="emphasisParamsInfoItemSubTitle--"]'));
|
||||
emphTitles.forEach(function(el, i) {{
|
||||
var name = emphSubs[i] ? emphSubs[i].textContent.trim() : null;
|
||||
if (name) attrs[name] = el.textContent.trim();
|
||||
}});
|
||||
var genTitles = Array.from(document.querySelectorAll('[class*="generalParamsInfoItemTitle--"]'));
|
||||
var genSubs = Array.from(document.querySelectorAll('[class*="generalParamsInfoItemSubTitle--"]'));
|
||||
genTitles.forEach(function(el, i) {{
|
||||
var val = genSubs[i] ? genSubs[i].textContent.trim() : null;
|
||||
if (val) attrs[el.textContent.trim()] = val;
|
||||
}});
|
||||
result.attributes = attrs;
|
||||
|
||||
var bodyText = document.body.textContent;
|
||||
var reviewMatch = bodyText.match(/评价[·：]\\s*([0-9,+万]+)/);
|
||||
result.reviewCount = reviewMatch ? reviewMatch[1] : null;
|
||||
|
||||
return JSON.stringify(result);
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{ error: true, message: e.message }});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,129 @@
|
||||
---
|
||||
name: taobao-product-reviews
|
||||
description: "Fetch customer reviews for a Taobao or Tmall product by itemId, returning reviewer name, date, purchased variant, review text, and photo URLs. Use when user asks to get product reviews from Taobao, scrape Taobao customer feedback, extract buyer reviews by item ID, collect Tmall ratings and comments, 采集淘宝商品评价, 抓取淘宝买家评论, 获取淘宝商品评论, 天猫商品评价抓取, 按商品ID获取评价. Also applies to sentiment analysis of product reviews, building review datasets, and monitoring product rating changes."
|
||||
---
|
||||
|
||||
# Taobao — Product Reviews
|
||||
|
||||
> itemId → paginated customer reviews (reviewer, date, purchased SKU, text, photos)
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Navigate to a Taobao/Tmall product page, load the reviews section, and extract customer review content.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Target page is already open in the browser: `https://item.taobao.com/item.htm?id={itemId}`
|
||||
- User is logged in to Taobao (user avatar or nickname visible in the page header)
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
### 2. Login Verification
|
||||
|
||||
If login status for Taobao has been confirmed in the current session → skip this step.
|
||||
|
||||
Otherwise: open `https://www.taobao.com` and observe the page header:
|
||||
- User nickname visible → logged in, continue execution
|
||||
- Login button visible → not logged in, inform the user that Taobao login is needed first, assist the user in completing the login flow
|
||||
|
||||
User refuses or cannot log in → terminate execution.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page, never bypassing authentication or access controls. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. `$(...)` is bash syntax; it is recommended to use the bash tool for execution.
|
||||
|
||||
### DOM: product reviews (data extraction)
|
||||
|
||||
The reviews section is lazy-loaded below the main product area. Follow these steps to load and extract reviews:
|
||||
|
||||
1. `navigate "https://item.taobao.com/item.htm?id={itemId}"`
|
||||
2. `wait stable`
|
||||
3. Close any popup: look for buttons with text "开心收下", "不了", "关闭" and click to dismiss
|
||||
4. Scroll to trigger lazy loading of the tabs/reviews section:
|
||||
`scroll down --amount 8000`
|
||||
5. `wait --selector "[class*='tabTitleItem--']" --state attached --timeout 10000`
|
||||
- If timeout: `scroll down --amount 8000` again and retry wait once more
|
||||
- If still no tabs after 2 attempts: take `screenshot` to confirm page state; the product page may be rendering in a condensed mode — check Known Limitations below
|
||||
6. `eval "$(python scripts/extract-reviews.py '{itemId}')"`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
[
|
||||
{
|
||||
"username": "一笑奈何",
|
||||
"date": "2026-06-03",
|
||||
"purchasedSku": "轻巧白|英转中转换器【适用国内电器】适用马来西亚/新加坡等国家",
|
||||
"content": "商品非常好，造工很用心！，还会再回购！",
|
||||
"photos": [
|
||||
"https://gw.alicdn.com/bao/uploaded/i1/O1CN015Cyg4b2FPR2YNq3PD_!!4611686018427383816-0-rate.jpg"
|
||||
],
|
||||
"rating": null
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Notes:
|
||||
- `purchasedSku`: the specific variant the reviewer purchased (extracted from "已购：{sku}" prefix in review header)
|
||||
- `content`: review text body; may be empty if reviewer submitted only photos
|
||||
- `photos`: review photo URLs; empty array if no photos
|
||||
- `rating`: star rating; not always visible in current page layout (null is common)
|
||||
- Reviews shown are the default sort (most recent or most helpful as determined by Taobao)
|
||||
|
||||
Error handling: if result count = 0 after scroll attempts, the reviews section may not have loaded in the current browser rendering environment. Try navigating to the product page fresh (`navigate` again) and repeating the scroll sequence. If still failing, this is a known rendering limitation — see Known Limitations below.
|
||||
|
||||
### DOM: paginate to next review page
|
||||
|
||||
After extracting current page reviews:
|
||||
|
||||
1. `eval "$(python scripts/next-review-page.py)"`
|
||||
- Returns `{"hasNext": true, "buttonText": "下一页"}` if next page exists, or `{"hasNext": false}` if on last page
|
||||
2. If `hasNext` is true: `state` to find the "下一页" button index → `click <index>`
|
||||
3. `wait stable`
|
||||
4. Re-run `eval "$(python scripts/extract-reviews.py '{itemId}')"`
|
||||
|
||||
## Enum Parameters
|
||||
|
||||
[collection failed] Sort/filter options for reviews (e.g., newest, most helpful): these controls exist in the reviews section UI but require the tabs section to be loaded; their URL parameters are not exposed and must be set via UI clicks on the sort tabs within the reviews section.
|
||||
|
||||
## Pagination
|
||||
|
||||
**DOM Pagination**: Click the "下一页" button in the reviews section footer. Each page shows ~10 reviews. Termination: "下一页" button is absent or `hasNext` returns false.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`result count >= 1` and `username` non-null rate = 100%
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- **Tab section lazy-loading**: The reviews section (along with all tabs: specs, images, recommendations) is lazy-loaded and requires scrolling past the main product area to appear. In some browser sessions or rendering environments, the tabs section does not load even after multiple scroll attempts. This is an intermittent behavior of the Taobao product page rendering engine and does not indicate a site change. Workaround: close and reopen the browser session, then navigate fresh.
|
||||
- Requires Taobao login; unauthenticated sessions redirect to login page
|
||||
- Review content is only visible on the product page; there is no standalone reviews URL for Taobao/Tmall products
|
||||
- Only shows positive buyer reviews by default; negative reviews may require clicking a filter tab within the reviews section (if visible)
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script to loop through itemIds serially within a single session; add 3–5 second intervals to allow the lazy-loaded reviews section to render.
|
||||
- **Test before batch execution**: After writing a batch script, you must first test with 1–2 items to verify the reviews section loads correctly; only then run the full batch. Never skip testing and execute in batch directly.
|
||||
- **Reduce redundant pre-operations**: When collecting multiple pages of reviews for one product, stay on the same page and paginate via button click rather than re-navigating.
|
||||
- **Error resumption**: Save results page by page; on failure, resume from the last successful page.
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/taobao-product-reviews.memory.md`
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file. Do not record what keywords were used or how many results were returned — those are task outputs, not experience.
|
||||
@@ -0,0 +1,82 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('item_id') # itemId (for documentation only; page already loaded)
|
||||
args = parser.parse_args()
|
||||
|
||||
js = f"""
|
||||
(function() {{
|
||||
try {{
|
||||
var commentItems = document.querySelectorAll('[class*="Comment--"]');
|
||||
if (!commentItems || commentItems.length === 0) {{
|
||||
return JSON.stringify({{ error: true, message: 'No review items found. Reviews section may not have loaded — try scrolling down further (scroll down --amount 8000) and waiting, then retry.' }});
|
||||
}}
|
||||
var reviews = Array.from(commentItems).map(function(item) {{
|
||||
var usernameEl = item.querySelector('[class*="userName--"]');
|
||||
var username = usernameEl ? usernameEl.textContent.trim() : null;
|
||||
|
||||
// Extract date and purchased SKU from comment header text
|
||||
// Format: "{{date}}已购：{{sku}}" or embedded in the header div
|
||||
var headerEl = item.querySelector('[class*="userInfo--"], [class*="header--"]');
|
||||
var headerText = headerEl ? headerEl.textContent.trim() : '';
|
||||
var dateMatch = headerText.match(/(\\d{{4}}-\\d{{2}}-\\d{{2}})/);
|
||||
var date = dateMatch ? dateMatch[1] : null;
|
||||
var skuMatch = headerText.match(/已购[：:](.*?)(?:\\n|$)/);
|
||||
var purchasedSku = skuMatch ? skuMatch[1].trim() : null;
|
||||
if (!purchasedSku) {{
|
||||
// Try extracting from item text after the username
|
||||
var fullText = item.textContent;
|
||||
var skuMatch2 = fullText.match(/已购[：:](.*?)(?=商品|\\d{{4}}-|$)/);
|
||||
purchasedSku = skuMatch2 ? skuMatch2[1].trim().slice(0, 150) : null;
|
||||
}}
|
||||
|
||||
// Extract review content text
|
||||
var contentEl = item.querySelector('[class*="content--"], [class*="reviewContent--"], [class*="reviewText--"]');
|
||||
var content = '';
|
||||
if (contentEl) {{
|
||||
content = contentEl.textContent.trim();
|
||||
}} else {{
|
||||
// Fallback: extract text that is not username/date/sku from the item
|
||||
var allText = item.textContent.trim();
|
||||
if (username) allText = allText.replace(username, '');
|
||||
if (date) allText = allText.replace(date, '');
|
||||
if (purchasedSku) allText = allText.replace('已购：' + purchasedSku, '').replace('已购:' + purchasedSku, '');
|
||||
content = allText.trim();
|
||||
}}
|
||||
|
||||
// Extract review photos
|
||||
var photoEls = item.querySelectorAll('img[src*="rate"], img[src*="-0-rate"], img[src*="oss"]');
|
||||
if (!photoEls || photoEls.length === 0) {{
|
||||
photoEls = item.querySelectorAll('img[src*="alicdn"], img[src*="gw.alicdn"]');
|
||||
}}
|
||||
var photos = Array.from(photoEls)
|
||||
.map(function(img) {{ return img.src; }})
|
||||
.filter(function(src) {{ return src && !src.includes('avatar') && !src.includes('logo'); }});
|
||||
|
||||
// Rating is often not shown as a visible number in current layout
|
||||
var ratingEl = item.querySelector('[class*="rating--"], [class*="star--"], [class*="score--"]');
|
||||
var rating = ratingEl ? ratingEl.textContent.trim() : null;
|
||||
|
||||
return {{
|
||||
username: username,
|
||||
date: date,
|
||||
purchasedSku: purchasedSku,
|
||||
content: content,
|
||||
photos: photos,
|
||||
rating: rating
|
||||
}};
|
||||
}});
|
||||
|
||||
return JSON.stringify(reviews);
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{ error: true, message: e.message }});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,33 @@
|
||||
import sys
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
|
||||
js = """
|
||||
(function() {
|
||||
try {
|
||||
// Look for the next page button in the reviews/comments section
|
||||
var allBtns = Array.from(document.querySelectorAll('button, a, div[class*="next"], div[class*="page"]'));
|
||||
var nextBtn = allBtns.find(function(el) {
|
||||
var text = el.textContent.trim();
|
||||
return text === '下一页' || text === '>' || text === '›';
|
||||
});
|
||||
if (!nextBtn) {
|
||||
return JSON.stringify({ hasNext: false });
|
||||
}
|
||||
var isDisabled = nextBtn.disabled ||
|
||||
nextBtn.getAttribute('aria-disabled') === 'true' ||
|
||||
nextBtn.className.indexOf('disabled') >= 0;
|
||||
return JSON.stringify({
|
||||
hasNext: !isDisabled,
|
||||
buttonText: nextBtn.textContent.trim()
|
||||
});
|
||||
} catch(e) {
|
||||
return JSON.stringify({ error: true, message: e.message });
|
||||
}
|
||||
})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,109 @@
|
||||
---
|
||||
name: taobao-shop-catalog
|
||||
description: "Browse a Taobao or Tmall shop's product catalog by shopId, returning paginated product listings with itemId and title. Use when user asks to scrape a Taobao shop, get all products from a store, list items in a Taobao/Tmall shop, fetch shop catalog by userId or shopId, 采集淘宝店铺商品, 抓取淘宝店铺所有商品, 获取天猫店铺商品列表, 淘宝店铺目录, 按店铺ID采集商品. Also applies to shop inventory monitoring, competitor store analysis, and bulk itemId collection from a specific seller."
|
||||
---
|
||||
|
||||
# Taobao — Shop Catalog
|
||||
|
||||
> shopId → paginated shop product listing (itemId, title, image URL)
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Navigate to a Taobao or Tmall shop's catalog page and extract product listings.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Target page is already open in the browser: `https://shop{shopId}.taobao.com/category.htm`
|
||||
- User is logged in to Taobao (user avatar or nickname visible in the page header)
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
### 2. Login Verification
|
||||
|
||||
If login status for Taobao has been confirmed in the current session → skip this step.
|
||||
|
||||
Otherwise: open `https://www.taobao.com` and observe the page header:
|
||||
- User nickname visible → logged in, continue execution
|
||||
- Login button visible → not logged in, inform the user that Taobao login is needed first, assist the user in completing the login flow
|
||||
|
||||
User refuses or cannot log in → terminate execution.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page, never bypassing authentication or access controls. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. `$(...)` is bash syntax; it is recommended to use the bash tool for execution.
|
||||
|
||||
### DOM: shop catalog product list (data extraction)
|
||||
|
||||
Navigate to the shop's catalog page, then extract:
|
||||
|
||||
1. `navigate "https://shop{shopId}.taobao.com/category.htm?search=y&pageNo={page}"`
|
||||
- Note: Tmall shops redirect to `https://{shopName}.tmall.com/category.htm?search=y&pageNo={page}`
|
||||
- The `search=y` parameter activates the paginated search mode
|
||||
2. `wait stable`
|
||||
3. `eval "$(python scripts/extract-catalog.py '{shopId}' --page {page})"`
|
||||
|
||||
Parameters:
|
||||
- `shopId`: numerical shop ID (e.g., `67095450`); found in shop URL as `shop{shopId}.taobao.com`
|
||||
- `--page`: page number, 1-based, default `1`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
[
|
||||
{
|
||||
"itemId": "1041516493508",
|
||||
"title": "绿联T8梯形排插插座转换器插线板大间距宿舍桌面充电多孔位插排",
|
||||
"imageUrl": "https://img.alicdn.com/imgextra/...",
|
||||
"itemUrl": "https://detail.tmall.com/item.htm?id=1041516493508"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Notes:
|
||||
- `price` is not included — Taobao shop catalog pages use font-based price obfuscation that cannot be decoded via DOM extraction. Use the `taobao-product-detail` skill to fetch prices for specific items.
|
||||
- `imageUrl` may be a lazy-loaded URL from `data-ks-lazyload-custom` attribute when the image has not scrolled into view
|
||||
|
||||
Error handling: if result count = 0, check that the page loaded correctly (`screenshot`), confirm the shopId is valid, and retry. Some shops may be Tmall-only and require following the redirect URL.
|
||||
|
||||
## Pagination
|
||||
|
||||
**URL Pagination**: URL pattern `https://shop{shopId}.taobao.com/category.htm?search=y&pageNo={N}` (or `https://{shopName}.tmall.com/category.htm?search=y&pageNo={N}` after redirect), increment `pageNo` from 1. Each page returns up to 60 items. Termination: when `result count = 0` or next page link `href` with `pageNo={N+1}` is absent from the DOM.
|
||||
|
||||
Next page link selector: `a[href*="pageNo"]` (contains the next page number).
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`result count >= 1` and `itemId` non-null rate = 100%
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Prices are font-obfuscated on the shop catalog page and cannot be extracted; fetch individual item prices via `taobao-product-detail`
|
||||
- Some shop categories are not shown in the default `category.htm` view; category-specific browsing requires clicking category links in the shop nav
|
||||
- Shop redirect from `shop{id}.taobao.com` to `{name}.tmall.com` changes the URL structure; the script handles both
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script to loop through pages serially; add 2–3 second intervals between navigations.
|
||||
- **Test before batch execution**: After writing a batch script, you must first test with 1–2 pages to verify the script runs correctly; only then run the full batch. Never skip testing and execute in batch directly.
|
||||
- **Reduce redundant pre-operations**: Stay in the same session for all pages of one shop without re-login checks.
|
||||
- **Error resumption**: Save results page by page during batch processing; on failure, resume from the last successful page rather than starting over.
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/taobao-shop-catalog.memory.md`
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file. Do not record what keywords were used or how many results were returned — those are task outputs, not experience.
|
||||
@@ -0,0 +1,60 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('shop_id') # shop ID (for documentation only; page already loaded)
|
||||
parser.add_argument('--page', default='1') # current page number
|
||||
args = parser.parse_args()
|
||||
|
||||
js = f"""
|
||||
(function() {{
|
||||
try {{
|
||||
var cards = document.querySelectorAll('.item');
|
||||
if (!cards || cards.length === 0) {{
|
||||
return JSON.stringify({{ error: true, message: 'No product cards found (.item). Page may not have loaded or shopId is invalid.' }});
|
||||
}}
|
||||
var items = Array.from(cards).map(function(card) {{
|
||||
var link = card.querySelector('a[href*="id="]');
|
||||
var href = link ? link.href : '';
|
||||
var idMatch = href.match(/[?&]id=([^&]+)/);
|
||||
var itemId = idMatch ? idMatch[1] : null;
|
||||
if (!itemId) return null;
|
||||
|
||||
var titleEl = card.querySelector('a.item-name, [class*="title"], .title');
|
||||
var title = titleEl ? titleEl.textContent.trim() : '';
|
||||
|
||||
var img = card.querySelector('img[src], img[data-ks-lazyload-custom]');
|
||||
var imgUrl = '';
|
||||
if (img) {{
|
||||
imgUrl = img.src || img.getAttribute('data-ks-lazyload-custom') || '';
|
||||
if (imgUrl && !imgUrl.startsWith('http')) imgUrl = 'https:' + imgUrl;
|
||||
}}
|
||||
|
||||
return {{
|
||||
itemId: itemId,
|
||||
title: title,
|
||||
imageUrl: imgUrl,
|
||||
itemUrl: 'https://item.taobao.com/item.htm?id=' + itemId
|
||||
}};
|
||||
}}).filter(function(item) {{ return item !== null; }});
|
||||
|
||||
// Deduplicate by itemId
|
||||
var seen = {{}};
|
||||
var unique = items.filter(function(item) {{
|
||||
if (seen[item.itemId]) return false;
|
||||
seen[item.itemId] = true;
|
||||
return true;
|
||||
}});
|
||||
|
||||
return JSON.stringify(unique);
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{ error: true, message: e.message }});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
---
|
||||
name: business-contact-social-links-skill
|
||||
description: "This skill helps users automatically extract official website and social media profiles. Agent should proactively apply this skill when users express needs like search for official website and social media contacts of a company, find YouTube and LinkedIn profiles by company name, extract social media links from a specific website URL, discover a company's X and Facebook presence, gather business contact details using their brand name, retrieve TikTok and Instagram links from a target website, track competitor social media strategy, extract multi-platform social links for lead generation, find official contact channels of local businesses, collect canonical profile URLs for outreach campaigns, or build a contact database from Yellow Pages leads."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Business Contact & Social Links Extractor Skill
|
||||
|
||||
## 📖 Brief
|
||||
This skill integrates two BrowserAct templates to cover two common scenarios. When the user provides a **company name**, it searches Google to extract the company's official website and social media channels. When the user provides a **website URL**, it scrapes the site to extract social media profile links (LinkedIn, Facebook, X/Twitter, Instagram, YouTube, TikTok). The script auto-detects the input type and routes to the correct template.
|
||||
|
||||
## ✨ Features
|
||||
1. **No Hallucinations**: Pre-set workflows avoid AI generative hallucinations, ensuring stable and precise data extraction.
|
||||
2. **No Captcha Issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP Restrictions**: No need to handle regional IP restrictions or geofencing.
|
||||
4. **Faster Execution**: Tasks execute faster compared to pure AI-driven browser automation solutions.
|
||||
5. **Cost-Effective**: Significantly lowers data acquisition costs compared to high-token-consuming AI solutions.
|
||||
|
||||
## 🔑 API Key Setup
|
||||
Before running, check the `BROWSERACT_API_KEY` environment variable. If not set, do not take other measures; ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key yet, please visit the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
The agent should determine whether the input is a company name or a website URL and pass it to the script. The script automatically routes the request to the correct template.
|
||||
|
||||
1. **Target (Company Name or Website URL)**
|
||||
- **Required**: Yes
|
||||
- **Type**: `string`
|
||||
- **Description**: The name of the company to search for, or the direct website URL to scrape social links from.
|
||||
- **Example**: `OpenAI` or `https://www.rotorooter.com/`
|
||||
|
||||
## 🚀 Invocation Method
|
||||
Agent should execute the following command to invoke the skill:
|
||||
|
||||
```bash
|
||||
# Search by company name
|
||||
python -u ./scripts/business_contact_social_links.py "OpenAI"
|
||||
|
||||
# Scrape by website URL
|
||||
python -u ./scripts/business_contact_social_links.py "https://www.rotorooter.com/"
|
||||
```
|
||||
|
||||
### ⏳ Execution Monitoring
|
||||
Since this task involves automated browser operations, it may take several minutes. The script outputs **timestamped status logs** continuously (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent guidelines**:
|
||||
- Monitor the terminal output while waiting.
|
||||
- As long as new status logs appear, the task is running normally; do not misjudge it as frozen.
|
||||
- Only consider triggering retry if the status remains unchanged for a long time or output stops without a final result.
|
||||
|
||||
## 📊 Data Output
|
||||
Upon successful execution, the script prints structured JSON data extracted from the API response.
|
||||
|
||||
**If a Company Name was provided**, the output includes:
|
||||
- `Company Name`: The official name of the company.
|
||||
- `Current Page URL`: The official website URL.
|
||||
- `Social Media Information`: A consolidated list of discovered social media profiles.
|
||||
|
||||
**If a Website URL was provided**, the output includes an array of objects:
|
||||
- `platform`: Platform name (e.g., LinkedIn, YouTube).
|
||||
- `url`: Canonical URL of the external profile.
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
If an error occurs during script execution (e.g., network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check Output Content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. **Do not retry**; guide the user to re-check and provide the correct API Key.
|
||||
- If the output **contains** `"concurrent"` or `"too many running tasks"`, it means the concurrent task limit has been reached. **Do not retry**; guide the user to upgrade their plan.
|
||||
**Agent must inform the user**:
|
||||
> "The current task cannot be executed because your BrowserAct account has reached the concurrent task limit. Please visit the [BrowserAct Plan Upgrade Page](https://www.browseract.com/reception/recharge) to upgrade your plan."
|
||||
- If the output **does not contain the above error keywords** but the task failed (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically re-execute the script once**.
|
||||
|
||||
2. **Retry Limit**:
|
||||
- Automatic retry is limited to **one time**. If the second attempt fails, stop retrying and report the specific error to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Sales Lead Enrichment**: Find the official website and social channels for a list of company names.
|
||||
2. **Influencer Outreach**: Extract all social media profiles from a creator's personal website.
|
||||
3. **Competitor Analysis**: Discover which social media platforms a rival company prioritizes.
|
||||
4. **CRM Data Cleaning**: Verify and populate missing social links and official URLs for existing leads.
|
||||
5. **Multi-Channel Marketing**: Build targeted lists of businesses with active social media presence.
|
||||
6. **Social Listening Setup**: Quickly gather official handles to monitor brand mentions.
|
||||
7. **B2B Prospecting**: Combine with Yellow Pages data to build complete business intelligence profiles.
|
||||
8. **Partner Vetting**: Evaluate the digital footprint of a potential business partner using just their name.
|
||||
9. **Directory Building**: Automate the collection of website and social links for a local business directory.
|
||||
10. **Brand Tracking**: Track all official contact channels of specific brands across the web.
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
TEMPLATE_ID_COMPANY = "89720224238575367"
|
||||
TEMPLATE_ID_URL = "64208669348207286"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
def run_business_contact_social_links_task(api_key, target):
|
||||
"""
|
||||
Starts a Business Contact or Social Links task and polls for completion.
|
||||
"""
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
|
||||
# Determine if target is a URL or a Company Name
|
||||
# Simple heuristic: if it starts with http/https, contains www., or has a dot without spaces
|
||||
target_lower = target.strip().lower()
|
||||
is_url = target_lower.startswith("http://") or target_lower.startswith("https://") or target_lower.startswith("www.") or ("." in target_lower and " " not in target_lower)
|
||||
|
||||
if is_url:
|
||||
# Use Social Links Scraper Template
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID_URL,
|
||||
"input_parameters": [
|
||||
{"name": "Website_URL", "value": target}
|
||||
]
|
||||
}
|
||||
print(f"Detected URL. Using Social Links Scraper template.", flush=True)
|
||||
else:
|
||||
# Use Google Business Contact Finder Template
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID_COMPANY,
|
||||
"input_parameters": [
|
||||
{"name": "Company_name", "value": target}
|
||||
]
|
||||
}
|
||||
print(f"Detected Company Name. Using Google Business Contact Finder template.", flush=True)
|
||||
|
||||
# 1. Start Task
|
||||
try:
|
||||
res = requests.post(
|
||||
f"{API_BASE_URL}/run-task-by-template",
|
||||
json=payload, headers=headers, timeout=30
|
||||
).json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
res_str = str(res)
|
||||
if "Invalid authorization" in res_str:
|
||||
print("Error: Invalid authorization. Please check your BrowserAct API Key.", flush=True)
|
||||
elif "concurrent" in res_str.lower() or "too many running tasks" in res_str.lower():
|
||||
print("Error: Concurrent task limit reached. Please upgrade your plan at https://www.browseract.com/reception/recharge", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 900
|
||||
poll_start = time.time()
|
||||
finished = False
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(
|
||||
f"{API_BASE_URL}/get-task-status?task_id={task_id}",
|
||||
headers=headers, timeout=30
|
||||
).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
finished = True
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
if not finished:
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(
|
||||
f"{API_BASE_URL}/get-task?task_id={task_id}",
|
||||
headers=headers, timeout=30
|
||||
).json()
|
||||
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python business_contact_social_links.py <target>", flush=True)
|
||||
print("target can be a Company Name (e.g., 'OpenAI') or a Website URL (e.g., 'https://www.rotorooter.com/')", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Provide it to me or set it as an environment variable (BROWSERACT_API_KEY).", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
target = sys.argv[1]
|
||||
|
||||
result = run_business_contact_social_links_task(api_key, target)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,116 @@
|
||||
---
|
||||
name: github-project-contributor-finder-api-skill
|
||||
description: "This skill helps users extract GitHub repository project details and contributor contact information using keywords, stars, and update dates. Agent should proactively apply this skill when users express needs like search for GitHub projects by keywords, find top open-source contributors in specific domains, extract developer contacts from GitHub repositories, discover trending repositories with high stars, gather contributor profiles and social links for tech recruiting, retrieve GitHub project descriptions and metrics, build developer communities by finding active contributors, search for repositories updated recently, collect personal website and Twitter links of developers, generate targeted leads for developer tools, or track active open-source contributors for collaboration."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# GitHub Project & Contributor Finder API Skill
|
||||
|
||||
## 📖 Brief
|
||||
This skill utilizes BrowserAct's GitHub Project & Contributor Finder API to extract project details and contributor contact information from GitHub. Simply provide keywords, minimum stars, and an update date filter — BrowserAct traverses the search results, extracts repository metrics, and fetches detailed contributor profiles, returning it all directly via API without building crawler scripts or dealing with rate limits.
|
||||
|
||||
## ✨ Features
|
||||
1. **No Hallucinations**: Pre-set workflows avoid AI generative hallucinations, ensuring stable and precise data extraction.
|
||||
2. **No Captcha Issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP Restrictions**: No need to handle regional IP restrictions or geofencing.
|
||||
4. **Faster Execution**: Tasks execute faster compared to pure AI-driven browser automation solutions.
|
||||
5. **Cost-Effective**: Significantly lowers data acquisition costs compared to high-token-consuming AI solutions.
|
||||
|
||||
## 🔑 API Key Setup
|
||||
Before running, check the `BROWSERACT_API_KEY` environment variable. If not set, do not take other measures; ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key yet, please visit the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
The agent should flexibly configure the following parameters based on user requirements:
|
||||
|
||||
1. **KeyWords**
|
||||
- **Type**: `string`
|
||||
- **Description**: Keywords for searching repositories.
|
||||
- **Example**: `browser automation`, `react framework`, `machine learning`
|
||||
- **Default**: `browser automation`
|
||||
|
||||
2. **stars**
|
||||
- **Type**: `number`
|
||||
- **Description**: Minimum number of stars the repository should have.
|
||||
- **Example**: `100`, `1000`
|
||||
- **Default**: `100`
|
||||
|
||||
3. **updated**
|
||||
- **Type**: `string`
|
||||
- **Description**: Filter repositories by the date they were last updated (format: `YYYY-MM-DD`).
|
||||
- **Example**: `2026-01-01`, `2025-06-01`
|
||||
- **Default**: `2026-01-01`
|
||||
|
||||
4. **Page_Turns**
|
||||
- **Type**: `number`
|
||||
- **Description**: Number of search result pages to paginate through. For example, if there are 39 pages and you want the first 2, input `2`.
|
||||
- **Example**: `1`, `2`
|
||||
- **Default**: `1`
|
||||
|
||||
5. **date_limit_per_page**
|
||||
- **Type**: `number`
|
||||
- **Description**: Number of data items to extract per page in the search results list.
|
||||
- **Example**: `5`, `10`
|
||||
- **Default**: `5`
|
||||
|
||||
## 🚀 Invocation Method
|
||||
Agent should execute the following command to invoke the skill:
|
||||
|
||||
```bash
|
||||
# Example invocation (all parameters)
|
||||
python -u ./scripts/github_project_contributor_finder_api.py "browser automation" 100 "2026-01-01" 1 5
|
||||
|
||||
# Minimal invocation (only keywords, others use defaults)
|
||||
python -u ./scripts/github_project_contributor_finder_api.py "react framework"
|
||||
```
|
||||
|
||||
### ⏳ Execution Monitoring
|
||||
Since this task involves automated browser operations, it may take several minutes. The script outputs **timestamped status logs** continuously (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent guidelines**:
|
||||
- Monitor the terminal output while waiting.
|
||||
- As long as new status logs appear, the task is running normally; do not misjudge it as frozen.
|
||||
- Only consider triggering retry if the status remains unchanged for a long time or output stops without a final result.
|
||||
|
||||
## 📊 Data Output
|
||||
Upon successful execution, the script parses and prints the structured results from the API response.
|
||||
|
||||
**Project Fields**:
|
||||
- `repository_name`: The name of the GitHub repository.
|
||||
- `repository_url`: The URL link to the repository.
|
||||
- `repository_owner_name`: The owner/creator of the repository.
|
||||
- `repository_description`: A brief description of the repository.
|
||||
- `star_count`: The number of stars the repository has received.
|
||||
|
||||
**Contributor Fields**:
|
||||
- `user_name`: The GitHub username of the contributor.
|
||||
- `profile_url`: The URL link to the contributor's profile.
|
||||
- `bio`: The bio or short description of the contributor.
|
||||
- `repositories_summary`: A summary of other repositories owned by the contributor.
|
||||
- `personal_website`: The contributor's personal website link.
|
||||
- `twitter`: The contributor's Twitter handle.
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
If an error occurs during script execution (e.g., network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check Output Content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. **Do not retry**; guide the user to re-check and provide the correct API Key.
|
||||
- If the output **contains** `"concurrent"` or `"too many running tasks"`, it means the concurrent task limit has been reached. **Do not retry**; guide the user to upgrade their plan.
|
||||
**Agent must inform the user**:
|
||||
> "The current task cannot be executed because your BrowserAct account has reached the concurrent task limit. Please visit the [BrowserAct Plan Upgrade Page](https://www.browseract.com/reception/recharge) to upgrade your plan."
|
||||
- If the output **does not contain the above error keywords** but the task failed (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically re-execute the script once**.
|
||||
|
||||
2. **Retry Limit**:
|
||||
- Automatic retry is limited to **one time**. If the second attempt fails, stop retrying and report the specific error to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Tech Recruiting**: Gather contributor profiles and social links from popular repositories to build candidate pipelines.
|
||||
2. **Open-Source Discovery**: Search for trending repositories by keywords and star count to find valuable projects.
|
||||
3. **Developer Outreach**: Collect personal websites and Twitter handles of active contributors for developer tool marketing.
|
||||
4. **Community Building**: Identify and connect with active open-source contributors in specific domains.
|
||||
5. **Competitor Analysis**: Monitor which developers contribute to competing projects.
|
||||
6. **Partnership Scouting**: Find repository owners for potential collaboration or sponsorship.
|
||||
7. **Market Research**: Analyze repository metrics and descriptions to understand technology trends.
|
||||
8. **Lead Generation**: Generate targeted leads for developer tools by finding projects with relevant tech stacks.
|
||||
9. **Academic Research**: Discover recently updated repositories in specific research areas.
|
||||
10. **Talent Mapping**: Build a database of skilled developers based on their GitHub contributions and profiles.
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
TEMPLATE_ID = "90284769428414983"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
|
||||
def run_github_project_contributor_finder_task(api_key, keywords, stars=100, updated="2026-01-01", page_turns=1, date_limit_per_page=5):
|
||||
"""
|
||||
Starts a GitHub Project & Contributor Finder task and polls for completion.
|
||||
Returns structured data as a string, or None on failure.
|
||||
"""
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "KeyWords", "value": keywords},
|
||||
{"name": "stars", "value": str(stars)},
|
||||
{"name": "updated", "value": updated},
|
||||
{"name": "Page_Turns", "value": str(page_turns)},
|
||||
{"name": "date_limit_per_page", "value": str(date_limit_per_page)}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Start Task", flush=True)
|
||||
try:
|
||||
res = requests.post(
|
||||
f"{API_BASE_URL}/run-task-by-template",
|
||||
json=payload, headers=headers, timeout=30
|
||||
).json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
res_str = str(res)
|
||||
if "Invalid authorization" in res_str:
|
||||
print("Error: Invalid authorization. Please check your BrowserAct API Key.", flush=True)
|
||||
elif "concurrent" in res_str.lower() or "too many running tasks" in res_str.lower():
|
||||
print("Error: Concurrent task limit reached. Please upgrade your plan at https://www.browseract.com/reception/recharge", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 900
|
||||
poll_start = time.time()
|
||||
finished = False
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(
|
||||
f"{API_BASE_URL}/get-task-status?task_id={task_id}",
|
||||
headers=headers, timeout=30
|
||||
).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
finished = True
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
if not finished:
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(
|
||||
f"{API_BASE_URL}/get-task?task_id={task_id}",
|
||||
headers=headers, timeout=30
|
||||
).json()
|
||||
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python github_project_contributor_finder_api.py <keywords> [stars] [updated] [page_turns] [date_limit_per_page]", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Provide it to me or set it as an environment variable (BROWSERACT_API_KEY).", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
keywords = sys.argv[1]
|
||||
stars = int(sys.argv[2]) if len(sys.argv) > 2 else 100
|
||||
updated = sys.argv[3] if len(sys.argv) > 3 else "2026-01-01"
|
||||
page_turns = int(sys.argv[4]) if len(sys.argv) > 4 else 1
|
||||
date_limit_per_page = int(sys.argv[5]) if len(sys.argv) > 5 else 5
|
||||
|
||||
result = run_github_project_contributor_finder_task(api_key, keywords, stars, updated, page_turns, date_limit_per_page)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,88 @@
|
||||
---
|
||||
name: google-maps-api-skill
|
||||
description: "This skill helps users automatically scrape business data from Google Maps using the BrowserAct Google Maps API. Agent should proactively trigger this skill for needs like finding restaurants in a specific city, extracting contact info of dental clinics, researching local competitors, collecting addresses of coffee shops, generating lead lists for specific industries, monitoring business ratings and reviews, getting opening hours of local services, finding specialized stores (e.g., Turkish-style restaurants), analyzing business categories in a region, extracting website links from local businesses, gathering phone numbers for sales outreach, mapping out service providers in a specific country."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Google Maps Automation Scraper Skill
|
||||
|
||||
## 📖 Introduction
|
||||
This skill leverages BrowserAct's Google Maps API template to provide a one-stop business data collection service. It extracts structured details directly from Google Maps, including business names, categories, contact info, ratings, and more. Simply provide the search keywords and location bias to get clean, actionable data.
|
||||
|
||||
## ✨ Features
|
||||
1. **No Hallucinations**: Pre-set workflows avoid AI generative hallucinations, ensuring stable and precise data extraction.
|
||||
2. **No Captcha Issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP Restrictions**: No need to handle regional IP restrictions or geofencing.
|
||||
4. **Faster Execution**: Tasks execute faster compared to pure AI-driven browser automation solutions.
|
||||
5. **Cost-Effective**: Significantly lowers data acquisition costs compared to high-token-consuming AI solutions.
|
||||
|
||||
## 🔑 API Key Setup
|
||||
Before running, check the `BROWSERACT_API_KEY` environment variable. If not set, do not take other measures; ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key, please visit the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
Configure the following parameters based on user requirements:
|
||||
|
||||
1. **keywords (Search Keywords)**
|
||||
- **Type**: `string`
|
||||
- **Description**: The query you would search for on Google Maps.
|
||||
- **Example**: `coffee shop`, `dental clinic`, `Turkish-style restaurant`
|
||||
|
||||
2. **language (UI Language)**
|
||||
- **Type**: `string`
|
||||
- **Description**: Defines the UI language and returned text language (e.g., en, zh-CN).
|
||||
- **Default**: `en`
|
||||
|
||||
3. **country (Country Bias)**
|
||||
- **Type**: `string`
|
||||
- **Description**: Specifies the country or region bias (e.g., us, gb, ca).
|
||||
- **Default**: `us`
|
||||
|
||||
## 🚀 Usage
|
||||
Execute the following script to get results in one command:
|
||||
|
||||
```bash
|
||||
# Example call
|
||||
python -u ./scripts/google_maps_api.py "keywords" "language" "country"
|
||||
```
|
||||
|
||||
### ⏳ Execution Monitoring
|
||||
Since this task involves automated browser operations, it may take some time (several minutes). The script will **continuously output status logs with timestamps** (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent Instructions**:
|
||||
- While waiting for the script result, keep monitoring the terminal output.
|
||||
- As long as the terminal is outputting new status logs, the task is running normally; do not mistake it for a deadlock or unresponsiveness.
|
||||
- Only if the status remains unchanged for a long time or the script stops outputting without returning a result should you consider triggering the retry mechanism.
|
||||
|
||||
## 📊 Data Output
|
||||
Upon success, the script parses and prints the following fields from the API:
|
||||
- `Title Name`: Official business name
|
||||
- `Category_primary`: Main business category
|
||||
- `Address`: Full street address
|
||||
- `Phone number`: Contact phone number
|
||||
- `Website link`: Official URL
|
||||
- `Rating`: Average star rating
|
||||
- `reviews_count`: Total number of reviews
|
||||
- `business_status`: Operational status (e.g., operational)
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
If an error occurs during script execution (e.g., network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check Output Content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. **Do not retry**; guide the user to re-check and provide the correct API Key.
|
||||
- If the output **does not contain** `"Invalid authorization"` but the task failed (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically try to re-execute the script once**.
|
||||
|
||||
2. **Retry Limit**:
|
||||
- Automatic retry is limited to **one time**. If the second attempt fails, stop retrying and report the specific error information to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Lead Generation**: Find "SaaS companies" in "us" for sales outreach.
|
||||
2. **Competitor Research**: Extract data on "coffee shops" in a specific neighborhood.
|
||||
3. **Market Analysis**: Identify the density of "dental clinics" in a region.
|
||||
4. **Contact Info Retrieval**: Get phone numbers and websites for "real estate agencies".
|
||||
5. **Local Service Discovery**: Find "Turkish-style restaurants" with high ratings.
|
||||
6. **Business Status Monitoring**: Check if specific stores are "operational".
|
||||
7. **Directory Building**: Gather addresses and categories for a local business directory.
|
||||
8. **Rating Benchmarking**: Compare ratings of various "luxury hotels".
|
||||
9. **Global Scouting**: Research "tech startups" in different countries like "gb" or "au".
|
||||
10. **Automated Data Sync**: Periodically pull local business data into a CRM.
|
||||
@@ -0,0 +1,110 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
TEMPLATE_ID = "77577579210625331"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
def run_google_maps_task(api_key, keywords, language="en", country="us"):
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "keywords", "value": keywords},
|
||||
{"name": "language", "value": language},
|
||||
{"name": "country", "value": country}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Start Task", flush=True)
|
||||
try:
|
||||
res = requests.post(f"{API_BASE_URL}/run-task-by-template", json=payload, headers=headers, timeout=30).json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
# Check for authorization error
|
||||
if "Invalid authorization" in str(res):
|
||||
print(f"Error: Invalid authorization. Please check your API key.", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 300
|
||||
poll_start = time.time()
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(f"{API_BASE_URL}/get-task-status?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(f"{API_BASE_URL}/get-task?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
|
||||
# The result is usually in output['string'] or within the raw JSON response
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Get API key from environment variable
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python google_maps_api.py <keywords> [language] [country]", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Provide it to me or set it as an environment variable (BROWSERACT_API_KEY).", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
keywords = sys.argv[1]
|
||||
language = sys.argv[2] if len(sys.argv) > 2 else "en"
|
||||
country = sys.argv[3] if len(sys.argv) > 3 else "us"
|
||||
|
||||
result = run_google_maps_task(api_key, keywords, language, country)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,261 @@
|
||||
---
|
||||
name: google-maps-contact-extract
|
||||
description: "Extracts business contact details from Google Maps search results and place detail pages, then visits each business website to collect emails, phone numbers, and social media profiles (Facebook, Instagram, Twitter/X, LinkedIn, YouTube, TikTok, Pinterest, Discord). Use when user mentions Google Maps contact extraction, maps email scraper, business lead generation from Google Maps, find emails from maps, scrape Google Maps businesses, maps business contacts, get phone from Google Maps, social media from maps listing, competitor research from maps, local business contact list, maps data export, google maps scraper, extract contacts from google maps, find business email google, gmaps leads, maps email finder, or wants to replicate Google Maps business data extraction."
|
||||
---
|
||||
|
||||
# Google Maps — Contact Extractor
|
||||
|
||||
> keyword + location → business list with name/address/phone/website + email/social media/contacts from each business's website
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Search Google Maps by keyword and location, collect place details from each result, visit each business's website to extract emails and social media profiles, and return a merged dataset replicating Google Maps Email Extractor functionality.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- For search: navigate to `https://www.google.com/maps/search/{keyword}/@{lat},{lng},{zoom}z` — the search results sidebar (feed) must be visible
|
||||
- For place detail: navigate to the individual place URL `https://www.google.com/maps/place/?q=place_id:{place_id}` — the place sidebar with name, address, phone must be visible
|
||||
- For website contact extraction: navigate to the business's website homepage
|
||||
- No login required for Google Maps or most business websites
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page, never bypassing authentication or access controls. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. It is recommended to use the bash tool for execution. **Working directory**: all `python scripts/` commands must be run from the Skill's root directory (the directory containing `SKILL.md` and `scripts/`). Use `cd {skill-directory}` before running batch scripts, or use absolute paths: `python {absolute-path-to-scripts}/xxx.py`.
|
||||
|
||||
### DOM: Search results list — extract business cards from Google Maps search feed
|
||||
|
||||
Prerequisite: navigate to and wait for Google Maps search results to load, then extract all visible business cards.
|
||||
|
||||
```
|
||||
navigate https://www.google.com/maps/search/{keyword}/@{lat},{lng},{zoom}z
|
||||
wait stable
|
||||
eval "$(python scripts/search-places.py '{keyword}' --max {max_count})"
|
||||
```
|
||||
|
||||
Parameters:
|
||||
- `{keyword}`: search term, e.g. `coffee shops`
|
||||
- `--max`: maximum number of places to extract, default `20`
|
||||
- `{lat},{lng}`: center coordinates, e.g. `40.7580,-73.9855`
|
||||
- `{zoom}`: zoom level, e.g. `14`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"keyword": "coffee shops",
|
||||
"count": 20,
|
||||
"places": [
|
||||
{
|
||||
"name": "Blue Dove Coffee",
|
||||
"rating": 4.8,
|
||||
"review_count": "292",
|
||||
"category": "Coffee shop",
|
||||
"price_range": null,
|
||||
"phone": null,
|
||||
"open_status": "Closed · Opens 7 AM",
|
||||
"place_id": "0xa5922b78cc420fb1:0x535506dc9cdb2cec",
|
||||
"lat": 40.7368708,
|
||||
"lng": -73.9909297,
|
||||
"maps_url": "https://www.google.com/maps/place/Blue+Dove+Coffee/data=..."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Pagination: Google Maps loads ~20 results per view. Scroll down in the results panel to trigger loading of additional results, then re-run the extraction script. Repeat until desired count is reached.
|
||||
|
||||
```
|
||||
scroll down --amount 3000
|
||||
wait stable
|
||||
eval "$(python scripts/search-places.py '{keyword}' --max {total_count})"
|
||||
```
|
||||
|
||||
### DOM: Place detail — extract full business info from a Google Maps place page
|
||||
|
||||
Prerequisite: navigate to the place page and wait for the sidebar to load with name, address, and phone visible.
|
||||
|
||||
```
|
||||
navigate https://www.google.com/maps/place/?q=place_id:{place_id}
|
||||
wait stable
|
||||
wait --selector "h1" --state visible --timeout 15000
|
||||
eval "$(python scripts/place-detail.py)"
|
||||
```
|
||||
|
||||
Alternatively, navigate directly using the `maps_url` returned from the search results component.
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"place_id": "0xa5922b78cc420fb1:0x535506dc9cdb2cec",
|
||||
"name": "Blue Dove Coffee",
|
||||
"rating": 4.8,
|
||||
"review_count": 292,
|
||||
"category": "Coffee shop",
|
||||
"address": "33 Union Square W, New York, NY 10003",
|
||||
"located_in": null,
|
||||
"phone": "(646) 939-0937",
|
||||
"website": "https://www.bluedovecoffee.com/",
|
||||
"menu_url": "https://order.dripos.com/Blue-Dove-Coffee",
|
||||
"price_range": "$1–10",
|
||||
"coordinates": { "lat": 40.7368708, "lng": -73.9909297 },
|
||||
"hours": [
|
||||
"Monday7 AM–5 PM",
|
||||
"Tuesday7 AM–5 PM",
|
||||
"Wednesday7 AM–5 PM",
|
||||
"Thursday7 AM–5 PM",
|
||||
"Friday7 AM–5 PM",
|
||||
"Saturday7 AM–5 PM",
|
||||
"Sunday7 AM–4 PM"
|
||||
],
|
||||
"service_options": ["Serves dine-in", "Offers takeout", "Offers delivery"],
|
||||
"amenities": ["Has wheelchair accessible entrance", "Accepts credit cards", "Accepts NFC mobile payments"],
|
||||
"maps_url": "https://www.google.com/maps/place/..."
|
||||
}
|
||||
```
|
||||
|
||||
### DOM: Website contacts — extract emails, phones, social media from a business website
|
||||
|
||||
Prerequisite: navigate to the business website homepage.
|
||||
|
||||
```
|
||||
navigate {website_url}
|
||||
wait stable
|
||||
eval "$(python scripts/extract-contacts.py --depth shallow)"
|
||||
```
|
||||
|
||||
For deeper extraction (also scans contact/about subpages), use `--depth deep`. After running with `--depth deep`, check `contact_subpages` in the output, then navigate to each and re-run:
|
||||
|
||||
```
|
||||
navigate {subpage_url}
|
||||
wait stable
|
||||
eval "$(python scripts/extract-contacts.py --depth shallow)"
|
||||
```
|
||||
|
||||
Parameters:
|
||||
- `--depth`: `shallow` (homepage only, default) or `deep` (also discovers and lists contact/about subpage URLs)
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"source_url": "https://www.bluedovecoffee.com/",
|
||||
"emails": ["support@bluedovecoffee.com", "careers@bluedovecoffee.com", "orders@bluedovecoffee.com"],
|
||||
"phone_numbers": [],
|
||||
"social_media": {
|
||||
"facebook": ["https://www.facebook.com/people/Blue-Dove-Coffee/100094867009022"],
|
||||
"instagram": ["https://www.instagram.com/bluedovecoffee"],
|
||||
"tiktok": ["https://www.tiktok.com/@bluedovecoffee"]
|
||||
},
|
||||
"contact_subpages": []
|
||||
}
|
||||
```
|
||||
|
||||
### Composite: Full pipeline — search + place details + website contacts
|
||||
|
||||
Complete flow replicating Google Maps Email Extractor:
|
||||
|
||||
1. Navigate to Google Maps search:
|
||||
```
|
||||
navigate https://www.google.com/maps/search/{keyword}/@{lat},{lng},{zoom}z
|
||||
wait stable
|
||||
eval "$(python scripts/search-places.py '{keyword}' --max {max_count})"
|
||||
```
|
||||
Save list of places (especially `name`, `maps_url`, `place_id`).
|
||||
|
||||
2. For each place: navigate to place detail page and extract full info:
|
||||
```
|
||||
navigate {maps_url}
|
||||
wait stable
|
||||
wait --selector "h1" --state visible --timeout 15000
|
||||
eval "$(python scripts/place-detail.py)"
|
||||
```
|
||||
Collect `website`, `phone`, `address`, `hours`, `rating`, etc.
|
||||
|
||||
3. For each place with a `website` value: extract contacts from the website:
|
||||
```
|
||||
navigate {website}
|
||||
wait stable
|
||||
eval "$(python scripts/extract-contacts.py --depth deep)"
|
||||
```
|
||||
For each URL in `contact_subpages`, navigate and extract again to find additional emails.
|
||||
|
||||
4. Merge results: join by `place_id` or `name`. Final record per business:
|
||||
```json
|
||||
{
|
||||
"name": "Blue Dove Coffee",
|
||||
"address": "33 Union Square W, New York, NY 10003",
|
||||
"phone": "(646) 939-0937",
|
||||
"website": "https://www.bluedovecoffee.com/",
|
||||
"rating": 4.8,
|
||||
"review_count": 292,
|
||||
"category": "Coffee shop",
|
||||
"coordinates": { "lat": 40.7368708, "lng": -73.9909297 },
|
||||
"hours": ["Monday7 AM–5 PM", "..."],
|
||||
"service_options": ["Serves dine-in", "Offers takeout"],
|
||||
"emails": ["support@bluedovecoffee.com"],
|
||||
"social_media": {
|
||||
"instagram": ["https://www.instagram.com/bluedovecoffee"],
|
||||
"tiktok": ["https://www.tiktok.com/@bluedovecoffee"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Pagination
|
||||
|
||||
**DOM Pagination**: Google Maps search loads ~20 results initially. Scroll down in the sidebar:
|
||||
```
|
||||
scroll down --amount 3000
|
||||
wait stable
|
||||
eval "$(python scripts/search-places.py '{keyword}' --max {total_count})"
|
||||
```
|
||||
Repeat until the desired number of results is reached. Termination: Google Maps typically shows up to ~120 results per search; the sidebar will show "You've reached the end of the results" or stop loading new items.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- `places.count >= 1` from search results extraction
|
||||
- Place detail returns `name != null AND (address != null OR phone != null)`
|
||||
- Website contact extraction returns `emails.length + phone_numbers.length + Object.keys(social_media).length >= 0` (no contacts = valid result for businesses without public contact info)
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Google Maps search results are limited to approximately 120 businesses per search query; to get more, use more specific sub-area searches
|
||||
- Phone numbers on the Google Maps search list card are not always shown; `place-detail.py` on the individual place page is more reliable
|
||||
- Website contact extraction depends on the business's website structure; sites using image-based emails or obfuscated text will return no emails
|
||||
- Some businesses do not have a website listed on Google Maps; those cannot be enriched with contact data
|
||||
- Google Maps may throttle requests if too many place pages are navigated in rapid succession; add 1–2 second delays between place navigations in batch scripts
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script that loops through the pipeline steps in a single session. Add a 1–2 second sleep between each place detail navigation to avoid triggering anti-scraping. Example:
|
||||
```bash
|
||||
for place_url in "${place_urls[@]}"; do
|
||||
browser-act --session gmaps-s1 navigate "$place_url"
|
||||
browser-act --session gmaps-s1 wait stable
|
||||
browser-act --session gmaps-s1 eval "$(python scripts/place-detail.py)"
|
||||
sleep 1.5
|
||||
done
|
||||
```
|
||||
- **Test before batch execution**: Test with 2–3 places before running the full batch
|
||||
- **Error resumption**: Save each place's result to a JSON file immediately after extraction; on failure, skip already-saved places by checking if the file exists
|
||||
- **Multiple sessions for throughput**: Open 2–3 parallel stealth browser sessions and distribute places across them; each session has an independent fingerprint
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/google-maps-contact-extract.memory.md`
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file.
|
||||
@@ -0,0 +1,114 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--depth', default='shallow') # 'shallow' = homepage only, 'deep' = also discover contact/about pages
|
||||
args = parser.parse_args()
|
||||
|
||||
depth = args.depth
|
||||
|
||||
js = r"""
|
||||
(function() {
|
||||
try {
|
||||
var html = document.documentElement.innerHTML;
|
||||
var url = window.location.href;
|
||||
var origin = window.location.origin;
|
||||
|
||||
// Extract emails from mailto links and visible text
|
||||
var emailSet = {};
|
||||
var mailtoRe = /mailto:([a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,})/gi;
|
||||
var emailRe = /\b([a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,})\b/g;
|
||||
var spamWords = ['example','pixel','sentry','schema.org','shopify','apple.com','wix.com',
|
||||
'google.com','amazon.com','cloudflare','w3.org','facebook.com','instagram.com',
|
||||
'twitter.com','tiktok.com','youtube.com','linkedin.com','pinterest.com','discord.com'];
|
||||
function isSpam(e) { return spamWords.some(function(w) { return e.indexOf(w) !== -1; }); }
|
||||
|
||||
var m;
|
||||
while ((m = mailtoRe.exec(html)) !== null) {
|
||||
var decoded = '';
|
||||
try { decoded = decodeURIComponent(m[1]).toLowerCase(); } catch(x) { decoded = m[1].toLowerCase(); }
|
||||
if (!isSpam(decoded)) emailSet[decoded] = true;
|
||||
}
|
||||
var bodyText = document.body ? (document.body.innerText || '') : '';
|
||||
while ((m = emailRe.exec(bodyText)) !== null) {
|
||||
var e = m[1].toLowerCase();
|
||||
if (!isSpam(e)) emailSet[e] = true;
|
||||
}
|
||||
var emails = Object.keys(emailSet).slice(0, 20);
|
||||
|
||||
// Extract social media profile links
|
||||
function extractSocial(pattern) {
|
||||
var re = new RegExp(pattern, 'gi');
|
||||
var found = {};
|
||||
var match;
|
||||
while ((match = re.exec(html)) !== null) {
|
||||
var link = match[0].split('"')[0].split("'")[0]
|
||||
.replace(/&/g, '&').replace(/[?#].*$/, '').replace(/\/$/, '');
|
||||
if (link.length < 150 && link.length > 15) found[link] = true;
|
||||
}
|
||||
return Object.keys(found).slice(0, 3);
|
||||
}
|
||||
|
||||
var socials = {};
|
||||
var fbLinks = extractSocial('https?://(?:www[.])?facebook[.]com/(?!sharer|share|dialog|tr[?]|plugins|embed|video|events|groups)[a-zA-Z0-9._/%-]+');
|
||||
var igLinks = extractSocial('https?://(?:www[.])?instagram[.]com/(?!p/|reel/|explore/|_/|share/|accounts/)[a-zA-Z0-9._]+/?');
|
||||
var twLinks = extractSocial('https?://(?:www[.])?(?:twitter|x)[.]com/(?!intent/|share[?]|hashtag|home|search|i/)[a-zA-Z0-9._]+/?');
|
||||
var liLinks = extractSocial('https?://(?:www[.])?linkedin[.]com/(?:company|in)/[a-zA-Z0-9._/-]+');
|
||||
var ytLinks = extractSocial('https?://(?:www[.])?youtube[.]com/(?:@|c/|channel/|user/)[a-zA-Z0-9._/-]+');
|
||||
var ttLinks = extractSocial('https?://(?:www[.])?tiktok[.]com/@[a-zA-Z0-9._/-]+');
|
||||
var piLinks = extractSocial('https?://(?:www[.])?pinterest[.]com/[a-zA-Z0-9._/-]+');
|
||||
var dcLinks = extractSocial('https?://(?:www[.])?discord[.](?:gg|com/invite)/[a-zA-Z0-9._/-]+');
|
||||
|
||||
if (fbLinks.length) socials.facebook = fbLinks;
|
||||
if (igLinks.length) socials.instagram = igLinks;
|
||||
if (twLinks.length) socials.twitter = twLinks;
|
||||
if (liLinks.length) socials.linkedin = liLinks;
|
||||
if (ytLinks.length) socials.youtube = ytLinks;
|
||||
if (ttLinks.length) socials.tiktok = ttLinks;
|
||||
if (piLinks.length) socials.pinterest = piLinks;
|
||||
if (dcLinks.length) socials.discord = dcLinks;
|
||||
|
||||
// Extract phone numbers from visible text
|
||||
var phoneRe = /(?:[+]?1[ .\-]?)?[(]?([0-9]{3})[)]?[ .\-]?([0-9]{3})[ .\-]?([0-9]{4})/g;
|
||||
var phones = {};
|
||||
while ((m = phoneRe.exec(bodyText)) !== null) {
|
||||
phones[m[0].trim()] = true;
|
||||
}
|
||||
var phone_numbers = Object.keys(phones).slice(0, 5);
|
||||
|
||||
// Find contact/about page links (only when --depth deep)
|
||||
var subpages = [];
|
||||
if ('__DEPTH__' === 'deep') {
|
||||
var contactLinks = Array.from(document.querySelectorAll('a[href]')).filter(function(a) {
|
||||
var href = a.href || '';
|
||||
var text = (a.textContent || '').toLowerCase();
|
||||
return (href.indexOf('/contact') !== -1 || href.indexOf('/about') !== -1 ||
|
||||
href.indexOf('/reach') !== -1 || text.match(/contact|about|reach us|get in touch/)) &&
|
||||
href.indexOf(origin) === 0 && href !== url;
|
||||
}).map(function(a) { return a.href.split('#')[0]; });
|
||||
var seen = {};
|
||||
contactLinks.forEach(function(l) { seen[l] = true; });
|
||||
subpages = Object.keys(seen).slice(0, 5);
|
||||
}
|
||||
|
||||
return JSON.stringify({
|
||||
source_url: url,
|
||||
emails: emails,
|
||||
phone_numbers: phone_numbers,
|
||||
social_media: socials,
|
||||
contact_subpages: subpages
|
||||
});
|
||||
} catch(e) {
|
||||
return JSON.stringify({ error: true, message: e.message });
|
||||
}
|
||||
})()
|
||||
""".replace('__DEPTH__', depth)
|
||||
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,109 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
args = parser.parse_args()
|
||||
|
||||
js = """
|
||||
(function() {
|
||||
try {
|
||||
var url = window.location.href;
|
||||
var placeIdM = url.match(/!1s([^!]+)!/);
|
||||
var coordM = url.match(/@(-?[0-9.]+),(-?[0-9.]+)/);
|
||||
var placeId = placeIdM ? placeIdM[1] : null;
|
||||
var lat = coordM ? parseFloat(coordM[1]) : null;
|
||||
var lng = coordM ? parseFloat(coordM[2]) : null;
|
||||
|
||||
// Validate we are on a place detail page (not the main maps page)
|
||||
var isPlacePage = url.indexOf('/maps/place/') !== -1 || url.indexOf('place_id') !== -1 || placeId;
|
||||
if (!isPlacePage) {
|
||||
return JSON.stringify({ error: true, message: 'Not on a place detail page. Navigate to a Google Maps place URL first.' });
|
||||
}
|
||||
var h1 = document.querySelector('h1');
|
||||
var name = h1 ? h1.textContent.trim() : null;
|
||||
if (!name) {
|
||||
return JSON.stringify({ error: true, message: 'Place detail not loaded yet. Make sure h1 is visible before running.' });
|
||||
}
|
||||
|
||||
var ratingEl = document.querySelector('[aria-label$=" stars "]') || document.querySelector('[aria-label*="stars"]');
|
||||
var ratingText = ratingEl ? ratingEl.getAttribute('aria-label') : '';
|
||||
var ratingM = ratingText.match(/([0-9.]+)/);
|
||||
var rating = ratingM ? parseFloat(ratingM[1]) : null;
|
||||
|
||||
var reviewEl = document.querySelector('[aria-label*="reviews"]');
|
||||
var reviewText = reviewEl ? reviewEl.getAttribute('aria-label') : '';
|
||||
var reviewM = reviewText.match(/([0-9,]+)/);
|
||||
var review_count = reviewM ? parseInt(reviewM[1].replace(',', '')) : null;
|
||||
|
||||
var catSpans = Array.from(document.querySelectorAll('[jsaction*="category"]'));
|
||||
var category = catSpans.length > 0 ? catSpans[0].textContent.trim() : null;
|
||||
|
||||
var addressEl = document.querySelector('[data-item-id="address"]');
|
||||
var address = addressEl ? addressEl.textContent.trim() : null;
|
||||
|
||||
var phoneEl = Array.from(document.querySelectorAll('[data-item-id]')).find(function(el) {
|
||||
return (el.getAttribute('data-item-id') || '').startsWith('phone:');
|
||||
});
|
||||
var phone = phoneEl ? phoneEl.textContent.trim() : null;
|
||||
|
||||
var websiteEl = document.querySelector('[data-item-id="authority"]');
|
||||
var website = websiteEl ? websiteEl.getAttribute('href') : null;
|
||||
|
||||
var menuEl = document.querySelector('[data-item-id="menu"]');
|
||||
var menu_url = menuEl ? menuEl.getAttribute('href') : null;
|
||||
|
||||
var priceEls = Array.from(document.querySelectorAll('[data-item-id]')).filter(function(el) {
|
||||
return (el.getAttribute('data-item-id') || '').match(/^[0-9]+$/);
|
||||
});
|
||||
var price_range = priceEls.length > 0 ? priceEls[0].textContent.trim() : null;
|
||||
|
||||
var hoursRows = Array.from(document.querySelectorAll('tr'))
|
||||
.map(function(tr) { return tr.textContent.trim(); })
|
||||
.filter(function(t) { return t.match(/Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday/); });
|
||||
var hours = hoursRows.length > 0 ? hoursRows : null;
|
||||
|
||||
var serviceOpts = Array.from(document.querySelectorAll('[aria-label*="ine-in"],[aria-label*="akeout"],[aria-label*="elivery"],[aria-label*="ickup"]'))
|
||||
.map(function(el) { return el.getAttribute('aria-label'); });
|
||||
|
||||
var aboutLabels = Array.from(document.querySelectorAll('[aria-label]')).map(function(el) {
|
||||
return el.getAttribute('aria-label');
|
||||
}).filter(function(l) {
|
||||
return l && l.length > 5 && l.length < 80 &&
|
||||
l.match(/^(Has|Accepts|No |Good for|Popular for|Casual|Cozy|Trendy|Upscale|Quick bite|Family|Serves great|Has great)/);
|
||||
});
|
||||
|
||||
var locatedIn = document.querySelector('[data-item-id="locatedin"]');
|
||||
var located_in = locatedIn ? locatedIn.textContent.trim() : null;
|
||||
|
||||
return JSON.stringify({
|
||||
place_id: placeId,
|
||||
name: name,
|
||||
rating: rating,
|
||||
review_count: review_count,
|
||||
category: category,
|
||||
address: address,
|
||||
located_in: located_in,
|
||||
phone: phone,
|
||||
website: website,
|
||||
menu_url: menu_url,
|
||||
price_range: price_range,
|
||||
coordinates: (lat && lng) ? { lat: lat, lng: lng } : null,
|
||||
hours: hours,
|
||||
service_options: serviceOpts.length > 0 ? serviceOpts : null,
|
||||
amenities: aboutLabels.length > 0 ? aboutLabels.slice(0, 20) : null,
|
||||
maps_url: url
|
||||
});
|
||||
} catch(e) {
|
||||
return JSON.stringify({ error: true, message: e.message });
|
||||
}
|
||||
})()
|
||||
"""
|
||||
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,66 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('keyword') # search keyword, e.g. "coffee shops"
|
||||
parser.add_argument('--max', default='20') # max number of places to extract
|
||||
args = parser.parse_args()
|
||||
|
||||
keyword = args.keyword.replace("'", "\\'")
|
||||
max_count = int(args.max)
|
||||
|
||||
js = r"""
|
||||
(function() {
|
||||
try {
|
||||
var cards = Array.from(document.querySelectorAll('[role=feed] > div'))
|
||||
.filter(function(el) { return el.querySelector('a[aria-label]'); });
|
||||
if (cards.length === 0) {
|
||||
return JSON.stringify({ error: true, message: 'No place cards found in feed. Make sure the Google Maps search results page is open with search results visible.' });
|
||||
}
|
||||
var results = cards.slice(0, __MAX__).map(function(el) {
|
||||
var link = el.querySelector('a');
|
||||
var nameEl = el.querySelector('[aria-label]');
|
||||
var t = el.textContent;
|
||||
|
||||
// Format: "Name 4.8(292) · $1-10Category · ... Address"
|
||||
var ratingM = t.match(/([0-9]\.[0-9])\(/);
|
||||
var reviewM = t.match(/\(([0-9,]+)\)/);
|
||||
var openM = t.match(/(Open|Closed)[^·\n]*/);
|
||||
var url = link ? link.href : null;
|
||||
var placeIdM = url ? url.match(/!1s([^!]+)!/) : null;
|
||||
var coordM = url ? url.match(/!3d(-?[0-9.]+)!4d(-?[0-9.]+)/) : null;
|
||||
var priceM = t.match(/\$[\d]+[–\-][\d]+|\$\$+/);
|
||||
var phoneM = t.match(/[(]?[0-9]{3}[)]?[ .\-][0-9]{3}[ .\-][0-9]{4}/);
|
||||
// Category follows price range or review count, then ends with " ·"
|
||||
var catM = t.match(/(?:\$[\d]+[^C\n]*?)([A-Z][a-zA-Z ]+(?:shop|restaurant|cafe|bar|hotel|store|bakery|pharmacy|gym|service|center|salon|spa|market))/i) ||
|
||||
t.match(/[·] ([A-Z][a-zA-Z ]*(?:shop|restaurant|cafe|bar|hotel|store|bakery|pharmacy|gym|salon|spa)) [·]/i);
|
||||
|
||||
return {
|
||||
name: nameEl ? nameEl.getAttribute('aria-label') : null,
|
||||
rating: ratingM ? parseFloat(ratingM[1]) : null,
|
||||
review_count: reviewM ? parseInt(reviewM[1].replace(',', '')) : null,
|
||||
category: catM ? (catM[1] || catM[0]).trim() : null,
|
||||
price_range: priceM ? priceM[0] : null,
|
||||
phone: phoneM ? phoneM[0] : null,
|
||||
open_status: openM ? openM[0].trim().slice(0, 60) : null,
|
||||
place_id: placeIdM ? placeIdM[1] : null,
|
||||
lat: coordM ? parseFloat(coordM[1]) : null,
|
||||
lng: coordM ? parseFloat(coordM[2]) : null,
|
||||
maps_url: url || null
|
||||
};
|
||||
});
|
||||
return JSON.stringify({ keyword: '__KW__', count: results.length, places: results });
|
||||
} catch(e) {
|
||||
return JSON.stringify({ error: true, message: e.message });
|
||||
}
|
||||
})()
|
||||
""".replace('__MAX__', str(max_count)).replace('__KW__', keyword)
|
||||
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
---
|
||||
name: google-maps-reviews-api-skill
|
||||
description: "This skill is designed to help users automatically extract reviews from Google Maps via the Google Maps Reviews API. Agent should proactively apply this skill when users request to find reviews for local businesses (e.g., coffee shops, clinics), monitor customer feedback for a specific brand or location, analyze sentiment of reviews for competitors, extract reviews for a chain of stores or services, track reputation of a local restaurant, gather user testimonials for a specific venue, conduct market research on service quality of local businesses, monitor reviews for a new retail location, collect feedback on public attractions or parks, identify common complaints for a specific service provider, research the best-rated places in a city, analyze recurring themes in reviews for a specific industry."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Google Maps Reviews Automation Skill
|
||||
|
||||
## 📖 Introduction
|
||||
This skill provides a one-stop review collection service using BrowserAct's Google Maps Reviews API template. It can extract structured review data directly from Google Maps search results. Simply provide the search keywords, language, and country to get clean, usable review data.
|
||||
|
||||
## ✨ Features
|
||||
1. **No Hallucinations**: Pre-set workflows avoid AI generative hallucinations, ensuring stable and precise data extraction.
|
||||
2. **No Captcha Issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP Restrictions**: No need to handle regional IP restrictions or geofencing.
|
||||
4. **Faster Execution**: Tasks execute faster compared to pure AI-driven browser automation solutions.
|
||||
5. **Cost-Effective**: Significantly lowers data acquisition costs compared to high-token-consuming AI solutions.
|
||||
|
||||
## 🔑 API Key Setup
|
||||
Before running, check the `BROWSERACT_API_KEY` environment variable. If not set, do not take other measures; ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key, please visit the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
The Agent should flexibly configure the following parameters when calling the script:
|
||||
|
||||
1. **KeyWords (Search Keywords)**
|
||||
- **Type**: `string`
|
||||
- **Description**: The query used to find places on Google Maps (e.g., business names, categories).
|
||||
- **Example**: `coffee shop`, `dental clinic`, `Tesla showroom`
|
||||
|
||||
2. **language (Language)**
|
||||
- **Type**: `string`
|
||||
- **Description**: Sets the UI language and the language of the returned text.
|
||||
- **Supported values**: `en`, `zh-CN`, `es`, `fr`, etc.
|
||||
- **Default**: `en`
|
||||
|
||||
3. **country (Country)**
|
||||
- **Type**: `string`
|
||||
- **Description**: Country or region bias for search results.
|
||||
- **Supported values**: `us`, `gb`, `ca`, `au`, `jp`, etc.
|
||||
- **Default**: `us`
|
||||
|
||||
## 🚀 Usage
|
||||
Agent should use the following independent script to achieve "one-line command result":
|
||||
|
||||
```bash
|
||||
# Example call
|
||||
python -u ./scripts/google_maps_reviews_api.py "Keywords" "Language" "Country"
|
||||
```
|
||||
|
||||
### ⏳ Execution Monitoring
|
||||
Since this task involves automated browser operations, it may take some time (several minutes). The script will **continuously output status logs with timestamps** (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent Instructions**:
|
||||
- While waiting for the script result, keep monitoring the terminal output.
|
||||
- As long as the terminal is outputting new status logs, the task is running normally; do not mistake it for a deadlock or unresponsiveness.
|
||||
- Only if the status remains unchanged for a long time or the script stops outputting without returning a result should you consider triggering the retry mechanism.
|
||||
|
||||
## 📊 Data Output
|
||||
After successful execution, the script parses and prints results from the API response:
|
||||
- `author_name`: Display name of the reviewer
|
||||
- `author_profile_url`: Profile URL of the reviewer
|
||||
- `rating`: Star rating
|
||||
- `text`: Review text content
|
||||
- `comment_date`: Human-readable date
|
||||
- `likes_count`: Number of likes
|
||||
- `author_image_url`: Reviewer's avatar URL
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
If an error occurs during script execution (e.g., network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check Output Content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. **Do not retry**; guide the user to re-check and provide the correct API Key.
|
||||
- If the output **does not contain** `"Invalid authorization"` but the task failed (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically try to re-execute the script once**.
|
||||
|
||||
2. **Retry Limit**:
|
||||
- Automatic retry is limited to **one time**. If the second attempt fails, stop retrying and report the specific error information to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Local Business Analysis**: Find reviews for cafes or clinics in a specific area.
|
||||
2. **Reputation Monitoring**: Track feedback for a specific brand location.
|
||||
3. **Competitive Benchmarking**: Analyze reviews of competitor stores.
|
||||
4. **Sentiment Analysis**: Gather review text for emotion and topic modeling.
|
||||
5. **Market Research**: Evaluate service quality across different regions.
|
||||
6. **Lead Qualification**: Use review data to identify high-quality service providers.
|
||||
7. **Customer Insight**: Understand recurring complaints or praises.
|
||||
8. **Venue Research**: Collect testimonials for parks, museums, or attractions.
|
||||
9. **Retail Monitoring**: Gather feedback for newly opened stores.
|
||||
10. **Service Quality Audit**: Analyze ratings and comments for a specific service chain.
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
TEMPLATE_ID = "77806855016940604"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
def run_google_maps_reviews_task(api_key, keywords, language="en", country="us"):
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "KeyWords", "value": keywords},
|
||||
{"name": "language", "value": language},
|
||||
{"name": "country", "value": country}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Start Task", flush=True)
|
||||
try:
|
||||
response = requests.post(f"{API_BASE_URL}/run-task-by-template", json=payload, headers=headers, timeout=30)
|
||||
if response.status_code == 401:
|
||||
print(f"Error: Invalid authorization. Please check your API key.", flush=True)
|
||||
return None
|
||||
res = response.json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
# Check for authorization error
|
||||
if "Invalid authorization" in str(res):
|
||||
print(f"Error: Invalid authorization. Please check your API key.", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 300
|
||||
poll_start = time.time()
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(f"{API_BASE_URL}/get-task-status?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(f"{API_BASE_URL}/get-task?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Prioritize environment variable
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python google_maps_reviews_api.py <keywords> [language] [country]", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Provide it to me or set it as an environment variable (BROWSERACT_API_KEY).", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
keywords = sys.argv[1]
|
||||
language = sys.argv[2] if len(sys.argv) > 2 else "en"
|
||||
country = sys.argv[3] if len(sys.argv) > 3 else "us"
|
||||
|
||||
result = run_google_maps_reviews_task(api_key, keywords, language, country)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,96 @@
|
||||
---
|
||||
name: google-maps-search-api-skill
|
||||
description: "This skill is designed to help users automatically extract business data from Google Maps search results. The Agent should proactively apply this skill when the user makes the following requests searching for coffee shops in a specific city, finding dentists or medical clinics nearby, tracking competitors' locations in a certain area, extracting business leads from Google Maps lists, gathering restaurant data for market research, finding hotels or accommodation options in a region, locating specific services like coworking spaces or gyms, monitoring new business openings in a neighborhood, collecting contact information and addresses for sales prospecting, analyzing price ranges and cuisines of local eateries, getting ratings and review counts for a list of businesses, exporting local business data into a CRM or database."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Google Maps Search Automation Skill
|
||||
|
||||
## 📖 Introduction
|
||||
This skill utilizes the BrowserAct Google Maps Search API template to provide a one-stop business data collection service. It extracts structured business results directly from Google Maps search lists. Simply provide search keywords, language, and country filters to get clean, usable business data.
|
||||
|
||||
## ✨ Features
|
||||
1. **No Hallucinations**: Pre-set workflows avoid AI generative hallucinations, ensuring stable and precise data extraction.
|
||||
2. **No Captcha Issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP Restrictions**: No need to handle regional IP restrictions or geofencing.
|
||||
4. **Faster Execution**: Tasks execute faster compared to pure AI-driven browser automation solutions.
|
||||
5. **Cost-Effective**: Significantly lowers data acquisition costs compared to high-token-consuming AI solutions.
|
||||
|
||||
## 🔑 API Key Setup
|
||||
Before running, check the `BROWSERACT_API_KEY` environment variable. If not set, do not take other measures; ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key, please visit the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
When calling the script, the Agent should flexibly configure the following parameters based on user needs:
|
||||
|
||||
1. **KeyWords**
|
||||
- **Type**: `string`
|
||||
- **Description**: The content you want to search on Google Maps. Can be business names, categories, or specific keywords.
|
||||
- **Example**: `coffee shop`, `dental clinic`, `coworking space`
|
||||
|
||||
2. **language**
|
||||
- **Type**: `string`
|
||||
- **Description**: Sets the UI language and the language of returned text fields.
|
||||
- **Common Values**: `en`, `de`, `fr`, `it`, `es`, `ja`, `zh-CN`, `zh-TW`
|
||||
- **Default**: `en`
|
||||
|
||||
3. **country**
|
||||
- **Type**: `string`
|
||||
- **Description**: Sets the country or region bias for search results.
|
||||
- **Example**: `us`, `gb`, `ca`, `au`, `de`, `fr`, `jp`
|
||||
- **Default**: `us`
|
||||
|
||||
4. **max_dates**
|
||||
- **Type**: `number`
|
||||
- **Description**: The maximum number of places to extract from the search results list.
|
||||
- **Default**: `100`
|
||||
|
||||
## 🚀 Usage
|
||||
The Agent should execute the following independent script to achieve "one-line command result":
|
||||
|
||||
```bash
|
||||
# Example call
|
||||
python -u ./scripts/google_maps_search_api.py "search keywords" "language" "country" count
|
||||
```
|
||||
|
||||
### ⏳ Execution Monitoring
|
||||
Since this task involves automated browser operations, it may take some time (several minutes). The script will **continuously output status logs with timestamps** (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent Instructions**:
|
||||
- While waiting for the script result, keep monitoring the terminal output.
|
||||
- As long as the terminal is outputting new status logs, the task is running normally; do not mistake it for a deadlock or unresponsiveness.
|
||||
- Only if the status remains unchanged for a long time or the script stops outputting without returning a result should you consider triggering the retry mechanism.
|
||||
|
||||
## 📊 Data Output
|
||||
After successful execution, the script parses and prints results directly from the API response. Fields include:
|
||||
- `name`: Business name
|
||||
- `full address`: Full address
|
||||
- `rating`: Star rating
|
||||
- `review count`: Number of reviews
|
||||
- `price range`: Price level
|
||||
- `cuisine type`: Business category
|
||||
- `amenity tags`: Features like Wi-Fi
|
||||
- `review snippet`: Short review text
|
||||
- `service options`: Service indicators (e.g., "Order online")
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
If an error occurs during script execution (e.g., network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check Output Content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. **Do not retry**; guide the user to re-check and provide the correct API Key.
|
||||
- If the output **does not contain** `"Invalid authorization"` but the task failed (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically try to re-execute the script once**.
|
||||
|
||||
2. **Retry Limit**:
|
||||
- Automatic retry is limited to **one time**. If the second attempt fails, stop retrying and report the specific error information to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Local Business Discovery**: Find all "Italian restaurants" in Manhattan.
|
||||
2. **Sales Lead Generation**: Extract a list of "real estate agencies" in London for prospecting.
|
||||
3. **Competitor Mapping**: Locate all "Starbucks" branches in a specific city to map competition.
|
||||
4. **Market Research**: Analyze "boutique hotels" in Paris, including their ratings and price ranges.
|
||||
5. **Contact Collection**: Gather addresses and names of "dental clinics" in Sydney.
|
||||
6. **Service Search**: Find "24-hour pharmacies" or "emergency plumbers" in a certain area.
|
||||
7. **Neighborhood Monitoring**: Track new "gyms" or "yoga studios" opening in a community.
|
||||
8. **Structured Data Export**: Export structured data of "car dealerships" for CRM integration.
|
||||
9. **Sentiment Analysis Prep**: Get review snippets and ratings for "popular tourist attractions".
|
||||
10. **Global Search Localization**: Use different language and country settings to research "tech hubs" globally.
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
TEMPLATE_ID = "77805072070738748"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
def run_google_maps_search_task(api_key, keywords, language="en", country="us", max_dates=100):
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "KeyWords", "value": keywords},
|
||||
{"name": "language", "value": language},
|
||||
{"name": "country", "value": country},
|
||||
{"name": "max_dates", "value": str(max_dates)}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Start Task", flush=True)
|
||||
try:
|
||||
res = requests.post(f"{API_BASE_URL}/run-task-by-template", json=payload, headers=headers, timeout=30).json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
# Check for authorization error
|
||||
if "Invalid authorization" in str(res):
|
||||
print(f"Error: Invalid authorization. Please check your BrowserAct API Key.", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started. ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
max_poll_time = 300
|
||||
poll_start = time.time()
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(f"{API_BASE_URL}/get-task-status?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
try:
|
||||
task_info = requests.get(f"{API_BASE_URL}/get-task?task_id={task_id}", headers=headers, timeout=30).json()
|
||||
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Prioritize command line API key, then environment variable
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python google_maps_search_api.py <keywords> [language] [country] [max_dates]", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Provide it to me or set it as an environment variable (BROWSERACT_API_KEY).", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
keywords = sys.argv[1]
|
||||
language = sys.argv[2] if len(sys.argv) > 2 else "en"
|
||||
country = sys.argv[3] if len(sys.argv) > 3 else "us"
|
||||
max_dates = sys.argv[4] if len(sys.argv) > 4 else 100
|
||||
|
||||
result = run_google_maps_search_task(api_key, keywords, language, country, max_dates)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,111 @@
|
||||
---
|
||||
name: google-social-media-finder
|
||||
description: "Searches Google to discover social media profiles associated with a person, brand, or username; returns platform name, profile URL, username, bio snippet, and follower count across X, Instagram, Facebook, LinkedIn, TikTok, YouTube, Pinterest, Reddit, Snapchat, Threads, and more. Use when user wants to find someone's social media accounts, look up social profiles, discover where a person is active online, find brand social media pages, search social accounts by name, track digital footprint, find influencer profiles, check a company's social presence, locate a public figure's profiles, social media lookup, social media finder, find accounts across platforms, social profile search, online presence discovery, who is this person on social media, what social media does X use, find username across platforms."
|
||||
---
|
||||
|
||||
# Google — Social Media Finder
|
||||
|
||||
> Name or brand → all social media profiles found on Google (platform, URL, username, bio, followers)
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Given a person's name, brand name, or username, search Google and return all matching social media profile results from known platforms.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- No login required — Google search is publicly accessible
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page, never bypassing authentication or access controls. Its role is equivalent to copy-pasting on the user's behalf — the data is already on screen, automation merely saves time. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. `$(...)` is bash syntax; it is recommended to use the bash tool for execution.
|
||||
|
||||
Below are all atomic capabilities discovered and verified during the exploration phase, listed by command template with parameters. Simply invoke them as needed — no need to read `scripts/*.py` source code or re-verify. Only inspect scripts when execution fails for troubleshooting. Combine freely as needed during execution.
|
||||
|
||||
### DOM: social media profile results (data extraction type)
|
||||
|
||||
Navigate to the Google search page for the target name, then extract all social media profile results.
|
||||
|
||||
**Step 1 — Navigate:**
|
||||
|
||||
```
|
||||
navigate https://www.google.com/search?q={name}+social+media
|
||||
```
|
||||
|
||||
Replace `{name}` with the person's or brand's name, using `+` in place of spaces (e.g., `Taylor+Swift`, `Elon+Musk`, `Nike`).
|
||||
|
||||
**Step 2 — Wait for page:**
|
||||
|
||||
```
|
||||
wait stable
|
||||
```
|
||||
|
||||
**Step 3 — Extract:**
|
||||
|
||||
```bash
|
||||
eval "$(python scripts/extract-social-profiles.py)"
|
||||
```
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"error": false,
|
||||
"count": 5,
|
||||
"results": [
|
||||
{
|
||||
"platform": "Instagram", // social media platform name
|
||||
"username": "taylorswift", // handle or page name shown alongside platform
|
||||
"url": "https://www.instagram.com/taylorswift/", // direct profile URL
|
||||
"title": "Taylor Swift (@taylorswift) • Instagram photos and videos", // page title
|
||||
"snippet": "274M followers · 0 following · 706 posts ...", // bio/description snippet from search result
|
||||
"followers": "超过 2.7亿位关注者" // follower count as displayed (language depends on browser locale)
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
On error: `{"error": true, "message": "..."}` — check that the browser navigated to a Google search page and `.tF2Cxc` result containers are present.
|
||||
|
||||
## Pagination
|
||||
|
||||
**URL Pagination**: URL pattern `https://www.google.com/search?q={name}+social+media&start={offset}`, where `offset = (page - 1) * 10` (page 1 → `start=0` or omit, page 2 → `start=10`, page 3 → `start=20`). Next page link: `a#pnnext`. Termination: `a#pnnext` is absent (last page reached) or no social media results returned.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`result count >= 1` and `platform` and `url` fields are non-null for every item
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Results depend on Google's index — newly created or low-traffic profiles may not appear
|
||||
- Follower count text is localized to the browser's display language (e.g., Chinese characters for a Chinese-locale stealth browser)
|
||||
- Google may show sub-pages of the same profile as separate results (e.g., both `/elonmusk` and `/elonmusk/with_replies` from X); deduplicate by base URL if needed
|
||||
- Google SERP layout changes occasionally; if `.tF2Cxc` stops matching, inspect page HTML for updated container class names
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script to loop through the command templates serially within a single session; do not parallelize within one browser (prone to triggering anti-scraping restrictions). Refer to rate information in "Known Limitations" above to add appropriate intervals. To increase throughput, open multiple stealth browser sessions and distribute work across them — each session has an independent fingerprint so rate limits apply per session
|
||||
- **Test before batch execution**: After writing a batch script, you must first test with 1-2 items to verify the script runs correctly; only then run the full batch. Never skip testing and execute in batch directly
|
||||
- **Reduce redundant pre-operations**: When multiple steps depend on the same prerequisite state, complete them in batch under that state to avoid repeatedly establishing the same state
|
||||
- **Error resumption**: Save results item by item during batch processing; on failure, resume from the breakpoint rather than starting over
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/social-media-finder-google-social-media-finder.memory.md` (working directory is determined by the Agent running the Skill, typically the project root or current working directory)
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file. Do not record what keywords were used or how many results were returned — those are task outputs, not experience.
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
args = parser.parse_args()
|
||||
|
||||
js = f"""
|
||||
(function() {{
|
||||
try {{
|
||||
var socialDomains = ['x.com','twitter.com','instagram.com','facebook.com',
|
||||
'linkedin.com','tiktok.com','youtube.com','pinterest.com','reddit.com',
|
||||
'snapchat.com','threads.net','weibo.com','vk.com','discord.com','twitch.tv',
|
||||
'github.com','medium.com','quora.com','tumblr.com','soundcloud.com'];
|
||||
|
||||
function isSocialDomain(url) {{
|
||||
try {{
|
||||
var host = new URL(url).hostname.replace(/^www\\./, '');
|
||||
return socialDomains.some(function(d) {{ return host === d || host.endsWith('.' + d); }});
|
||||
}} catch(e) {{ return false; }}
|
||||
}}
|
||||
|
||||
function getPlatformName(url) {{
|
||||
try {{
|
||||
var host = new URL(url).hostname.replace(/^www\\./, '');
|
||||
var known = {{
|
||||
'x.com': 'X', 'twitter.com': 'Twitter', 'instagram.com': 'Instagram',
|
||||
'facebook.com': 'Facebook', 'linkedin.com': 'LinkedIn', 'tiktok.com': 'TikTok',
|
||||
'youtube.com': 'YouTube', 'pinterest.com': 'Pinterest', 'reddit.com': 'Reddit',
|
||||
'snapchat.com': 'Snapchat', 'threads.net': 'Threads', 'weibo.com': 'Weibo',
|
||||
'vk.com': 'VK', 'discord.com': 'Discord', 'twitch.tv': 'Twitch',
|
||||
'github.com': 'GitHub', 'medium.com': 'Medium', 'quora.com': 'Quora',
|
||||
'tumblr.com': 'Tumblr', 'soundcloud.com': 'SoundCloud'
|
||||
}};
|
||||
return known[host] || host.split('.')[0];
|
||||
}} catch(e) {{ return ''; }}
|
||||
}}
|
||||
|
||||
var containers = document.querySelectorAll('.tF2Cxc');
|
||||
if (!containers.length) {{
|
||||
return JSON.stringify({{
|
||||
error: true,
|
||||
message: 'No search result containers found (.tF2Cxc). Page may not have loaded or Google changed its layout.'
|
||||
}});
|
||||
}}
|
||||
|
||||
var items = [];
|
||||
containers.forEach(function(el) {{
|
||||
var link = el.querySelector('a.zReHs');
|
||||
var url = link ? link.href : '';
|
||||
if (!url || !isSocialDomain(url)) return;
|
||||
|
||||
var platformUser = el.querySelector('.VuuXrf') ? el.querySelector('.VuuXrf').textContent.trim() : '';
|
||||
var parts = platformUser.split(/\\s*\\u00b7\\s*/);
|
||||
|
||||
var title = el.querySelector('h3.LC20lb') ? el.querySelector('h3.LC20lb').textContent.trim() : null;
|
||||
var snippet = el.querySelector('.VwiC3b') ? el.querySelector('.VwiC3b').textContent.trim() : null;
|
||||
var followers = el.querySelector('cite.qLRx3b') ? el.querySelector('cite.qLRx3b').textContent.trim() : null;
|
||||
|
||||
items.push({{
|
||||
platform: getPlatformName(url),
|
||||
username: parts[1] ? parts[1].trim() : '',
|
||||
url: url,
|
||||
title: title,
|
||||
snippet: snippet,
|
||||
followers: followers
|
||||
}});
|
||||
}});
|
||||
|
||||
return JSON.stringify({{ error: false, count: items.length, results: items }});
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{ error: true, message: e.message }});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,208 @@
|
||||
---
|
||||
name: indeed-job-search
|
||||
description: "Scrape job listings from Indeed.com by keyword, location, and country. Returns job title, company, salary, rating, description, benefits, and apply links. Use when user mentions Indeed, Indeed scraper, Indeed jobs, scrape Indeed, job search Indeed, Indeed job listings, extract Indeed data, Indeed job data, get jobs from Indeed, Indeed employment data, job market research Indeed, Indeed salary data, bulk job extraction Indeed, Indeed job scraper, monitor Indeed listings, Indeed hiring data, job postings Indeed, Indeed career search. Also applies to: job market analysis, salary benchmarking from Indeed, competitor hiring monitoring, recruitment data collection, building job databases from Indeed search results."
|
||||
---
|
||||
|
||||
# Indeed — Job Search
|
||||
|
||||
> keyword + location + country → structured job listing data (title, company, salary, description, benefits, apply link)
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Extract structured job listing data from Indeed search results including full job descriptions, salary information, company details, and application links.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Target page is already open in the browser: `https://www.indeed.com/jobs?q={keyword}&l={location}`
|
||||
- For pagination beyond page 1: user must be logged into Indeed (login/sign-in button is NOT visible, user avatar or account menu IS visible)
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
### 2. Login Verification (for pagination beyond page 1)
|
||||
|
||||
If login status for Indeed has been confirmed in the current session → skip this step.
|
||||
|
||||
Otherwise: open Indeed and observe the page login status:
|
||||
- Sign out entry, user avatar, or account menu exists → logged in, continue execution
|
||||
- Sign in/Register entry exists with no sign out entry → not logged in, inform the user that login is needed for pagination beyond page 1, assist the user in completing the login flow
|
||||
|
||||
User refuses or cannot log in → can still extract page 1 results (up to 24 jobs per search), but cannot paginate.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page, never bypassing authentication or access controls. Its role is equivalent to copy-pasting on the user's behalf — the data is already on screen, automation merely saves time. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. `$(...)` is bash syntax; it is recommended to use the bash tool for execution.
|
||||
|
||||
Below are all atomic capabilities discovered and verified during the exploration phase, listed by command template with parameters. Simply invoke them as needed — no need to read `scripts/*.py` source code or re-verify. Only inspect scripts when execution fails for troubleshooting. Combine freely as needed during execution.
|
||||
|
||||
### API: Extract job list from search results page
|
||||
|
||||
`eval "$(python scripts/search-jobs.py --max-items {max_items})"`
|
||||
|
||||
Parameters:
|
||||
- --max-items: Maximum number of items to return from current page, 0 for all (default: 0)
|
||||
|
||||
Prerequisites: Browser must be on an Indeed search results page (`indeed.com/jobs?q=...`). Navigate first:
|
||||
1. `navigate https://www.indeed.com/jobs?q={keyword}&l={location}` (add `&start={offset}` for pagination)
|
||||
2. `wait stable`
|
||||
3. If page title contains "Security Check" → `solve-captcha` → `wait stable`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"pageNumber": 1,
|
||||
"totalResults": 24,
|
||||
"results": [
|
||||
{
|
||||
"id": "8c06afbaf73fc880",
|
||||
"positionName": "Software Engineer",
|
||||
"company": "Google",
|
||||
"location": "San Francisco, CA",
|
||||
"postedAt": "3 days ago",
|
||||
"salary": "$120,000 - $180,000 a year",
|
||||
"salaryMin": 120000,
|
||||
"salaryMax": 180000,
|
||||
"salaryCurrency": "USD",
|
||||
"salaryType": "YEARLY",
|
||||
"jobType": ["Full-time"],
|
||||
"rating": 4.3,
|
||||
"reviewsCount": 15000,
|
||||
"companyLogo": "https://d2q79iu7y748jz.cloudfront.net/s/_squarelogo/256x256/...",
|
||||
"snippet": "<ul>...<b>software</b>...</ul>",
|
||||
"sponsored": false,
|
||||
"expired": false,
|
||||
"newJob": true,
|
||||
"url": "https://www.indeed.com/viewjob?jk=8c06afbaf73fc880",
|
||||
"externalApplyLink": null
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### API: Fetch full job detail by job key
|
||||
|
||||
`eval "$(python scripts/fetch-job-detail.py '{jobkey}')"`
|
||||
|
||||
Parameters:
|
||||
- jobkey: Indeed job key (the `id` field from search results)
|
||||
|
||||
Prerequisites: Browser must be on the same Indeed search results page where the job was found (the fetch uses same-origin cookies). Do NOT navigate away between search and detail fetch.
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"id": "a39828c8395d5dfe",
|
||||
"positionName": "Sr. Application Developer",
|
||||
"company": "University of California San Francisco",
|
||||
"location": "San Francisco, CA 94158",
|
||||
"rating": 4,
|
||||
"reviewsCount": 709,
|
||||
"companyLogo": "https://d2q79iu7y748jz.cloudfront.net/s/_squarelogo/256x256/...",
|
||||
"salary": "$101,300 - $190,000 a year",
|
||||
"salaryMin": 101300,
|
||||
"salaryMax": 190000,
|
||||
"salaryCurrency": "USD",
|
||||
"salaryType": "YEARLY",
|
||||
"jobType": "Full-time",
|
||||
"description": "<p><b>Job Description:</b></p><p>...(full HTML)...</p>",
|
||||
"benefits": ["Health insurance", "401(k)", "Dental insurance"],
|
||||
"postedAt": "13 days ago",
|
||||
"isExpired": false, "url": "https://www.indeed.com/viewjob?jk=a39828c8395d5dfe"
|
||||
}
|
||||
```
|
||||
|
||||
### Composite: Full job search with descriptions
|
||||
|
||||
Complete workflow to extract job listings with full descriptions:
|
||||
|
||||
1. `navigate https://www.indeed.com/jobs?q={keyword}&l={location}` → `wait stable`
|
||||
2. If title contains "Security Check" → `solve-captcha` → `wait stable`
|
||||
3. `eval "$(python scripts/search-jobs.py)"` → get job list with IDs
|
||||
4. For each job `id` from step 3:
|
||||
- `eval "$(python scripts/fetch-job-detail.py '{id}')"` → get full description
|
||||
5. Merge: search results provide base data, detail fetch adds `description`, `benefits`, enriched `salary`/`rating`
|
||||
|
||||
For pagination (requires login):
|
||||
- Repeat steps 1-4 with `&start=10`, `&start=20`, `&start=30`, etc.
|
||||
- Termination: when search-jobs.py returns fewer results than expected or `error: true`
|
||||
|
||||
Output: Combined array where each item has all fields from both search-jobs and fetch-job-detail.
|
||||
|
||||
## Enum Parameters
|
||||
|
||||
[API] country — supported country codes for Indeed search (append to URL as domain variant `https://{country-domain}/jobs?q=...`):
|
||||
|
||||
| Code | Domain | Country |
|
||||
|------|--------|---------|
|
||||
| US | www.indeed.com | United States |
|
||||
| GB | uk.indeed.com | United Kingdom |
|
||||
| CA | ca.indeed.com | Canada |
|
||||
| AU | au.indeed.com | Australia |
|
||||
| IN | in.indeed.com | India |
|
||||
| DE | de.indeed.com | Germany |
|
||||
| FR | fr.indeed.com | France |
|
||||
| JP | jp.indeed.com | Japan |
|
||||
| BR | br.indeed.com | Brazil |
|
||||
| MX | mx.indeed.com | Mexico |
|
||||
| SG | sg.indeed.com | Singapore |
|
||||
| NL | nl.indeed.com | Netherlands |
|
||||
| IT | it.indeed.com | Italy |
|
||||
| ES | es.indeed.com | Spain |
|
||||
| SE | se.indeed.com | Sweden |
|
||||
| CH | ch.indeed.com | Switzerland |
|
||||
| HK | hk.indeed.com | Hong Kong |
|
||||
| KR | kr.indeed.com | South Korea |
|
||||
| PL | pl.indeed.com | Poland |
|
||||
| AT | at.indeed.com | Austria |
|
||||
| BE | be.indeed.com | Belgium |
|
||||
| NZ | nz.indeed.com | New Zealand |
|
||||
|
||||
For other countries, use format: `{country-code-lowercase}.indeed.com`
|
||||
|
||||
## Pagination
|
||||
|
||||
**URL Pagination**: Parameter `start`, type: page-number (offset-based). Start value: `0` (page 1, implicit). Next page: increment by 10 (`start=10`, `start=20`, `start=30`...). Termination: search-jobs.py returns `error: true` with message about no results, or returns fewer than expected items. Note: each page actually returns ~15-24 items despite the offset increment of 10.
|
||||
|
||||
**Login requirement**: Page 2+ (`start=10` and above) redirects to `secure.indeed.com/auth` if not logged in.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- `result count >= 1` from search-jobs.py
|
||||
- Core fields non-null: `id`, `positionName`, `company`, `location` present for all results
|
||||
- `description` field from fetch-job-detail.py is non-null HTML string with length > 100
|
||||
- Data matches page content (job titles correspond to visible listings)
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Pagination beyond page 1 requires Indeed login (unauthenticated access limited to first page, ~24 results)
|
||||
- Indeed may trigger captcha/security check on first visit; `solve-captcha` resolves this
|
||||
- Job detail fetch (`fetch-job-detail.py`) must execute from within the search results page context (same-origin requirement)
|
||||
- Indeed's anti-bot detection may block access after excessive requests; use appropriate intervals (2-5 seconds between detail fetches)
|
||||
- Some fields (salary, jobType, benefits) may be null when not provided by the employer
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script to loop through the command templates serially within a single session; do not parallelize within one browser (prone to triggering anti-scraping restrictions). Add 2-3 second intervals between detail fetches. To increase throughput, open multiple stealth browser sessions and distribute work across them — each session has an independent fingerprint so rate limits apply per session
|
||||
- **Test before batch execution**: After writing a batch script, you must first test with 1-2 items to verify the script runs correctly; only then run the full batch. Never skip testing and execute in batch directly
|
||||
- **Reduce redundant pre-operations**: Extract all job IDs from the search page first (one eval), then batch-fetch details — avoid re-navigating to the search page between fetches
|
||||
- **Error resumption**: Save results item by item during batch processing; on failure, resume from the breakpoint rather than starting over
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `browser-act-skill-forge-memories/indeed-jobs-scraper-indeed-job-search.memory.md` (working directory is determined by the Agent running the Skill, typically the project root or current working directory)
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file. Do not record what keywords were used or how many results were returned — those are task outputs, not experience.
|
||||
@@ -0,0 +1,58 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('jobkey', help='Indeed job key (jk parameter)')
|
||||
args = parser.parse_args()
|
||||
|
||||
jobkey = args.jobkey
|
||||
|
||||
js = f"""(async function() {{
|
||||
try {{
|
||||
const jk = '{jobkey}';
|
||||
const url = '/viewjob?jk=' + jk + '&viewtype=embedded&spa=1';
|
||||
const resp = await fetch(url);
|
||||
if (!resp.ok) return JSON.stringify({{error: true, message: "HTTP " + resp.status}});
|
||||
const data = await resp.json();
|
||||
if (data.status !== 'success' || !data.body) return JSON.stringify({{error: true, message: "Unexpected response: " + (data.status || "unknown")}});
|
||||
const body = data.body;
|
||||
const info = body.jobInfoWrapperModel && body.jobInfoWrapperModel.jobInfoModel;
|
||||
const header = info && info.jobInfoHeaderModel;
|
||||
const salary = body.salaryInfoModel;
|
||||
const benefits = body.benefitsModel;
|
||||
const hiring = body.hiringInsightsModel;
|
||||
const metaHeader = info && info.jobMetadataHeaderModel;
|
||||
const description = info ? info.sanitizedJobDescription : null;
|
||||
const result = {{
|
||||
id: jk,
|
||||
positionName: header ? header.jobTitle : null,
|
||||
company: header ? header.companyName : null,
|
||||
location: header ? header.formattedLocation : null,
|
||||
rating: header && header.companyReviewModel && header.companyReviewModel.ratingsModel ? header.companyReviewModel.ratingsModel.rating : null,
|
||||
reviewsCount: header && header.companyReviewModel && header.companyReviewModel.ratingsModel ? header.companyReviewModel.ratingsModel.count : null,
|
||||
companyLogo: header && header.companyImagesModel ? header.companyImagesModel.logoUrl : null,
|
||||
salary: salary ? salary.salaryText : null,
|
||||
salaryMin: salary ? salary.salaryMin : null,
|
||||
salaryMax: salary ? salary.salaryMax : null,
|
||||
salaryCurrency: salary ? salary.salaryCurrency : null,
|
||||
salaryType: salary ? salary.salaryType : null,
|
||||
jobType: metaHeader ? metaHeader.jobType : null,
|
||||
description: description || null,
|
||||
benefits: benefits && benefits.benefits ? benefits.benefits.map(function(b) {{ return b.label; }}) : null,
|
||||
postedAt: hiring ? hiring.age : null,
|
||||
isExpired: body.isJobExpired || false,
|
||||
url: "https://www.indeed.com/viewjob?jk=" + jk
|
||||
}};
|
||||
return JSON.stringify(result);
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{error: true, message: e.message}});
|
||||
}}
|
||||
}})()"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,58 @@
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--max-items', type=int, default=0, help='Max items to return, 0 for all')
|
||||
args = parser.parse_args()
|
||||
|
||||
max_items = args.max_items
|
||||
|
||||
js = f"""
|
||||
(async function() {{
|
||||
try {{
|
||||
const pd = window.mosaic && window.mosaic.providerData;
|
||||
if (!pd) return JSON.stringify({{error: true, message: "mosaic.providerData not found - page may not be a valid Indeed search results page"}});
|
||||
const jc = pd['mosaic-provider-jobcards'];
|
||||
if (!jc) return JSON.stringify({{error: true, message: "mosaic-provider-jobcards not found in providerData"}});
|
||||
const model = jc.metaData && jc.metaData.mosaicProviderJobCardsModel;
|
||||
if (!model) return JSON.stringify({{error: true, message: "mosaicProviderJobCardsModel not found"}});
|
||||
const results = model.results;
|
||||
if (!results || results.length === 0) return JSON.stringify({{error: true, message: "No job results found on this page"}});
|
||||
const maxItems = {max_items};
|
||||
const items = (maxItems > 0 ? results.slice(0, maxItems) : results).map(r => ({{
|
||||
id: r.jobkey,
|
||||
positionName: r.displayTitle || null,
|
||||
company: r.company || null,
|
||||
location: r.formattedLocation || null,
|
||||
postedAt: r.formattedRelativeTime || null,
|
||||
salary: r.salarySnippet ? r.salarySnippet.text : null,
|
||||
salaryMin: r.extractedSalary ? r.extractedSalary.min : null,
|
||||
salaryMax: r.extractedSalary ? r.extractedSalary.max : null,
|
||||
salaryCurrency: r.salarySnippet ? r.salarySnippet.currency : null,
|
||||
salaryType: r.extractedSalary ? r.extractedSalary.type : null,
|
||||
jobType: r.jobTypes && r.jobTypes.length > 0 ? r.jobTypes : null,
|
||||
rating: r.companyRating || null,
|
||||
reviewsCount: r.companyReviewCount || null,
|
||||
companyLogo: r.companyBrandingAttributes ? r.companyBrandingAttributes.logoUrl : null,
|
||||
snippet: r.snippet || null,
|
||||
sponsored: !!r.sponsored,
|
||||
expired: !!r.expired,
|
||||
newJob: !!r.newJob,
|
||||
url: r.jobkey ? "https://www.indeed.com/viewjob?jk=" + r.jobkey : null,
|
||||
externalApplyLink: r.thirdPartyApplyUrl || null
|
||||
}}));
|
||||
return JSON.stringify({{pageNumber: model.pageNumber, totalResults: items.length, results: items}});
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{error: true, message: e.message}});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,99 @@
|
||||
---
|
||||
name: industry-key-contact-radar-api-skill
|
||||
description: "This skill helps users discover key contacts across industries, roles, and social platforms via the BrowserAct API. Agent should proactively apply this skill when users express needs like finding public profiles for founders or CEOs, discovering key decision-makers in a specific industry, extracting contact details for lead generation, searching for growth leaders on LinkedIn or Facebook, gathering professional networking profiles, retrieving URLs and names of industry leaders, finding marketing managers in a specific field, conducting competitive analysis by identifying key personnel, sourcing talent acquisition targets across platforms, or compiling a list of target roles on specific social sites."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Industry Key Contact Radar
|
||||
|
||||
## 📖 Brief
|
||||
This skill provides a one-stop contact discovery service using BrowserAct's Industry Key Contact Radar API template. It extracts structured contact details directly from search results across various platforms, including profile URLs, names, introductions, and company associations. Simply input an industry, result limit, target site, and job title to receive clean, actionable contact data.
|
||||
|
||||
## ✨ Features
|
||||
1. **No hallucinations, ensuring stable and accurate data extraction**: Pre-set workflows avoid AI generative hallucinations.
|
||||
2. **No CAPTCHA issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP restrictions or geo-blocking**: No need to handle regional IP restrictions or geofencing.
|
||||
4. **Faster execution**: Tasks execute faster compared to purely AI-driven browser automation solutions.
|
||||
5. **Extremely high cost-efficiency**: Significantly reduces data acquisition costs compared to AI solutions that consume massive amounts of tokens.
|
||||
|
||||
## 🔑 API Key Guide
|
||||
Before running, you must check the `BROWSERACT_API_KEY` environment variable. If it is not set, do not take other actions first; you should ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key yet, please go to the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
Agent should flexibly configure the following parameters based on user needs:
|
||||
|
||||
1. **industry (Industry)**
|
||||
- **Type**: `string`
|
||||
- **Description**: The industry or field to search for contacts.
|
||||
- **Example**: `Browser automation`, `E-commerce`, `Healthcare`
|
||||
- **Default**: `Browser automation`
|
||||
- **Required**: Yes
|
||||
|
||||
2. **Datelimit (Max Items)**
|
||||
- **Type**: `number`
|
||||
- **Description**: Maximum number of records to extract. The default value is recommended for the first run.
|
||||
- **Example**: `10`
|
||||
- **Default**: `10`
|
||||
|
||||
3. **site (Target Site)**
|
||||
- **Type**: `string`
|
||||
- **Description**: The social platform or website to search on.
|
||||
- **Example**: `facebook.com`, `linkedin.com`, `github.com`
|
||||
- **Default**: `facebook.com`
|
||||
- **Required**: Yes
|
||||
|
||||
4. **Job_Title (Job Title)**
|
||||
- **Type**: `string`
|
||||
- **Description**: The specific role or job title of the target contact.
|
||||
- **Example**: `founder`, `CEO`, `marketing manager`
|
||||
- **Default**: `founder`
|
||||
- **Required**: Yes
|
||||
|
||||
## 🚀 Invocation Method
|
||||
The Agent should execute the following independent script to achieve "one command gets results":
|
||||
|
||||
```bash
|
||||
# Example
|
||||
python -u ./scripts/industry_key_contact_radar_api.py "Browser automation" 10 "facebook.com" "founder"
|
||||
```
|
||||
|
||||
### ⏳ Running Status Monitoring
|
||||
Since this task involves automated browser operations, it may take a long time (several minutes). The script will **continuously output status logs with timestamps** while running (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent guidelines**:
|
||||
- While waiting for the script to return results, please keep an eye on the terminal output.
|
||||
- As long as the terminal continues to output new status logs, it means the task is running normally. Do not misjudge it as a deadlock or unresponsiveness.
|
||||
- If the status remains unchanged for a long time or the script stops outputting without returning a result, only then consider triggering the retry mechanism.
|
||||
|
||||
## 📊 Data Output
|
||||
After successful execution, the script will parse and print the results directly from the API response. The results include:
|
||||
- `url`: Direct link to the contact's public profile
|
||||
- `name`: The name of the contact or profile title
|
||||
- `Introduction`: A brief introduction or bio description of the contact
|
||||
- `Company`: The company or organization associated with the contact
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
During script execution, if errors occur (such as network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check the output content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. At this point, **do not retry**, but guide the user to recheck and provide the correct API Key.
|
||||
- If the output **contains** `"concurrent"` or `"too many running tasks"` or similar concurrency limit messages, it means the concurrent task limit for the current subscription plan has been reached. **Do not retry**; guide the user to upgrade their plan.
|
||||
**Agent must inform the user**:
|
||||
> "The current task cannot be executed because your BrowserAct account has reached the limit of concurrent tasks. Please go to the [BrowserAct Plan Upgrade Page](https://www.browseract.com/reception/recharge) to upgrade your subscription plan and enjoy more concurrent task benefits."
|
||||
- If the output **does not contain** the above error keywords but the task fails (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically try to run the script once more**.
|
||||
|
||||
2. **Retry limit**:
|
||||
- Automatic retry is limited to **once**. If the second attempt still fails, stop retrying and report the specific error message to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Lead Generation**: Finding public profiles for founders and CEOs in target industries.
|
||||
2. **Talent Acquisition**: Sourcing growth leaders and key roles for recruiting.
|
||||
3. **Competitive Analysis**: Identifying key personnel in competing organizations.
|
||||
4. **Professional Networking**: Gathering profiles for industry connections.
|
||||
5. **Market Intelligence**: Collecting URLs and names of industry leaders.
|
||||
6. **Cross-Platform Discovery**: Searching target profiles on specific sites like LinkedIn or GitHub.
|
||||
7. **B2B Outreach**: Finding decision-makers and extracting contact details.
|
||||
8. **Brand Marketing**: Locating marketing managers in a specific field.
|
||||
9. **Sales Prospecting**: Building lists of specific job titles across industries.
|
||||
10. **Targeted Job Title Search**: Compiling a list of target roles on specific social sites.
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for standard output and error streams
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# API Configuration
|
||||
TEMPLATE_ID = "88313898888855812"
|
||||
API_BASE_URL = "https://api.browseract.com/v2/workflow"
|
||||
|
||||
def run_industry_key_contact_radar_task(api_key, industry, date_limit=10, site="facebook.com", job_title="founder"):
|
||||
"""
|
||||
Starts an Industry Key Contact Radar task and polls for completion.
|
||||
|
||||
Args:
|
||||
api_key (str): BrowserAct API Key
|
||||
industry (str): The industry or field to search for contacts
|
||||
date_limit (int): Maximum number of records to extract
|
||||
site (str): The social platform or website to search on
|
||||
job_title (str): The specific role or job title of the target contact
|
||||
"""
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
payload = {
|
||||
"workflow_template_id": TEMPLATE_ID,
|
||||
"input_parameters": [
|
||||
{"name": "industry", "value": str(industry)},
|
||||
{"name": "Datelimit", "value": str(date_limit)},
|
||||
{"name": "site", "value": str(site)},
|
||||
{"name": "Job_Title", "value": str(job_title)}
|
||||
]
|
||||
}
|
||||
|
||||
# 1. Start Task
|
||||
print(f"Starting task via BrowserAct API...", flush=True)
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{API_BASE_URL}/run-task-by-template",
|
||||
json=payload, headers=headers, timeout=30
|
||||
)
|
||||
res = response.json()
|
||||
except Exception as e:
|
||||
print(f"Error: Connection to API failed - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if "id" not in res:
|
||||
res_str = str(res)
|
||||
if "Invalid authorization" in res_str:
|
||||
print("Error: Invalid authorization. Please check your BrowserAct API Key.", flush=True)
|
||||
elif "concurrent" in res_str.lower() or "too many running tasks" in res_str.lower():
|
||||
print("Error: Concurrent task limit reached. Please upgrade your plan at https://www.browseract.com/reception/recharge", flush=True)
|
||||
else:
|
||||
print(f"Error: Could not start task. Response: {res}", flush=True)
|
||||
return None
|
||||
|
||||
task_id = res["id"]
|
||||
print(f"Task started successfully. Task ID: {task_id}", flush=True)
|
||||
|
||||
# 2. Poll for Completion
|
||||
print(f"Waiting for task completion...", flush=True)
|
||||
max_poll_time = 900
|
||||
poll_start = time.time()
|
||||
finished = False
|
||||
|
||||
while time.time() - poll_start < max_poll_time:
|
||||
try:
|
||||
status_res = requests.get(
|
||||
f"{API_BASE_URL}/get-task-status?task_id={task_id}",
|
||||
headers=headers, timeout=30
|
||||
).json()
|
||||
status = status_res.get("status")
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Task Status: {status}", flush=True)
|
||||
|
||||
if status == "finished":
|
||||
print(f"[{timestamp}] Task finished successfully.", flush=True)
|
||||
finished = True
|
||||
break
|
||||
elif status in ["failed", "canceled"]:
|
||||
print(f"Error: Task {status}. Please check your BrowserAct dashboard.", flush=True)
|
||||
return None
|
||||
except Exception as e:
|
||||
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
print(f"[{timestamp}] Polling error: {e}. Retrying...", flush=True)
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
if not finished:
|
||||
print(f"Error: Task polling timed out after {max_poll_time} seconds.", flush=True)
|
||||
return None
|
||||
|
||||
# 3. Get Results
|
||||
print(f"Retrieving results...", flush=True)
|
||||
try:
|
||||
task_info_response = requests.get(
|
||||
f"{API_BASE_URL}/get-task?task_id={task_id}",
|
||||
headers=headers, timeout=30
|
||||
)
|
||||
task_info = task_info_response.json()
|
||||
|
||||
output = task_info.get("output", {})
|
||||
result_string = output.get("string")
|
||||
|
||||
if result_string:
|
||||
return result_string
|
||||
else:
|
||||
return json.dumps(task_info, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to retrieve results - {e}", flush=True)
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Get API Key from environment variable
|
||||
api_key = os.getenv("BROWSERACT_API_KEY")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python industry_key_contact_radar_api.py <industry> [date_limit] [site] [job_title]", flush=True)
|
||||
print("Example: python industry_key_contact_radar_api.py \"Browser automation\" 10 \"facebook.com\" \"founder\"", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not api_key:
|
||||
print("\n[!] ERROR: BrowserAct API Key is missing.", flush=True)
|
||||
print("Please follow these steps:", flush=True)
|
||||
print("1. Go to: https://www.browseract.com/reception/integrations", flush=True)
|
||||
print("2. Copy your API Key.", flush=True)
|
||||
print("3. Set it as an environment variable (BROWSERACT_API_KEY) or provide it in the chat.", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
industry = sys.argv[1]
|
||||
date_limit = int(sys.argv[2]) if len(sys.argv) > 2 else 10
|
||||
site = sys.argv[3] if len(sys.argv) > 3 else "facebook.com"
|
||||
job_title = sys.argv[4] if len(sys.argv) > 4 else "founder"
|
||||
|
||||
result = run_industry_key_contact_radar_task(api_key, industry, date_limit, site, job_title)
|
||||
if result:
|
||||
print(result, flush=True)
|
||||
@@ -0,0 +1,205 @@
|
||||
---
|
||||
name: linkedin-jobs-search
|
||||
description: "Search LinkedIn job listings and extract full job details. Supports filtering by work type (remote/on-site/hybrid), contract type (full-time/part-time/contract/internship), experience level, date posted, and company. Returns job title, company, location, work type, contract type, experience level, posted date, applicant count, job description, salary, and direct job URLs. Use when user mentions linkedin jobs, linkedin job search, scrape linkedin jobs, extract linkedin job listings, find jobs on linkedin, job openings, job postings linkedin, linkedin career search, job hunting linkedin, linkedin vacancy, jobs remote linkedin, work from home jobs linkedin, linkedin scraper jobs, linkedin job data, linkedin hiring, collect job leads linkedin."
|
||||
---
|
||||
|
||||
# LinkedIn — Job Search
|
||||
|
||||
> keywords + location + filters → paginated job list with full details
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Search LinkedIn job listings with full filter support, extract complete job data with full field coverage.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- The browser is open and the LinkedIn session is active (logged in). A LinkedIn jobs search page such as `https://www.linkedin.com/jobs/search/` must have been visited at least once so the CSRF token cookie is set.
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
### 2. Login Verification
|
||||
|
||||
If login status for LinkedIn has been confirmed in the current session → skip this step.
|
||||
|
||||
Otherwise: open `https://www.linkedin.com` and observe the page:
|
||||
- User avatar or "Me" menu visible → logged in, continue
|
||||
- Sign in / Join button visible → not logged in, inform user that LinkedIn login is required first
|
||||
|
||||
User refuses or cannot log in → terminate execution.
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It accesses LinkedIn through the user's logged-in browser, only reading data already available to the user. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py {params})"`. `$(...)` is bash syntax; it is recommended to use the bash tool for execution.
|
||||
|
||||
### API: Search LinkedIn jobs (list page)
|
||||
|
||||
`eval "$(python scripts/search-jobs.py '{keywords}' '{location}' --count {count} --start {start} --work-type {work_type} --job-type {job_type} --experience {experience} --time-posted {time_posted} --company-ids {company_ids})"`
|
||||
|
||||
Parameters:
|
||||
- `keywords`: job title or search keywords (e.g., `software engineer`, `data analyst`)
|
||||
- `location`: location name (e.g., `United States`, `New York`, `San Francisco Bay Area`)
|
||||
- `--count`: results per API call, default `25`, max `100`
|
||||
- `--start`: pagination offset, default `0`. Increment by `count` for each page
|
||||
- `--work-type`: work arrangement filter — `1`=On-site, `2`=Remote, `3`=Hybrid (optional)
|
||||
- `--job-type`: contract type filter — `F`=Full-time, `P`=Part-time, `C`=Contract, `T`=Temporary, `I`=Internship, `V`=Volunteer (optional)
|
||||
- `--experience`: experience level filter — `1`=Internship, `2`=Entry, `3`=Associate, `4`=Mid-Senior, `5`=Director (optional)
|
||||
- `--time-posted`: recency filter — `r86400`=24h, `r604800`=7 days, `r2592000`=30 days (optional)
|
||||
- `--company-ids`: comma-separated LinkedIn company numeric IDs (optional, e.g., `76987811,1441`)
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"total": 36015,
|
||||
"start": 0,
|
||||
"count": 5,
|
||||
"jobs": [
|
||||
{
|
||||
"id": "4416832078",
|
||||
"title": "Lead Frontend Software Engineer",
|
||||
"company": "RowsOne",
|
||||
"location": "Boca Raton, FL",
|
||||
"workType": "Remote",
|
||||
"jobUrl": "https://www.linkedin.com/jobs/view/4416832078",
|
||||
"companyUrl": "https://www.linkedin.com/company/rowsone"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Error handling: If `{"error": true}` is returned, check that the browser is still logged in to LinkedIn and navigate to `https://www.linkedin.com/jobs/search/` to refresh the session, then retry once.
|
||||
|
||||
### API: Get full job details
|
||||
|
||||
`eval "$(python scripts/job-detail.py '{job_id}')"`
|
||||
|
||||
Parameters:
|
||||
- `job_id`: numeric LinkedIn job posting ID (from `id` field in search results)
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"id": "4416832078",
|
||||
"title": "Lead Frontend Software Engineer",
|
||||
"company": "RowsOne",
|
||||
"companyUrl": "https://www.linkedin.com/company/rowsone",
|
||||
"location": "Boca Raton, FL",
|
||||
"workType": "Remote",
|
||||
"contractType": "Full-time",
|
||||
"experienceLevel": "Mid-Senior level",
|
||||
"listedAt": "2026-05-26T16:14:30.000Z",
|
||||
"applicantCount": 37,
|
||||
"description": "Lead Frontend Engineer (React / Next.js)...",
|
||||
"salary": null,
|
||||
"jobUrl": "https://www.linkedin.com/jobs/view/4416832078"
|
||||
}
|
||||
```
|
||||
|
||||
Error handling: HTTP 404 means job has been removed or ID is invalid. If `{"error": true, "message": "HTTP 403"}`, the LinkedIn session may have expired — navigate back to LinkedIn and verify login, then retry.
|
||||
|
||||
### Composite: Full job extraction (search list + detail for each job)
|
||||
|
||||
For complete output with all fields (description, contract type, experience level, posted date):
|
||||
|
||||
1. Run search component to collect job IDs and basic info
|
||||
2. For each job ID, run the detail component
|
||||
3. Merge results by job ID
|
||||
|
||||
Batch script template (bash):
|
||||
```bash
|
||||
#!/bin/bash
|
||||
SESSION="fb_explore"
|
||||
KEYWORDS="software engineer"
|
||||
LOCATION="United States"
|
||||
TOTAL_ROWS=50
|
||||
COUNT=25
|
||||
OUTPUT_FILE="output/jobs.jsonl"
|
||||
|
||||
offset=0
|
||||
collected=0
|
||||
while [ $collected -lt $TOTAL_ROWS ]; do
|
||||
batch_count=$((TOTAL_ROWS - collected))
|
||||
[ $batch_count -gt $COUNT ] && batch_count=$COUNT
|
||||
|
||||
result=$(browser-act --session $SESSION eval "$(python scripts/search-jobs.py "$KEYWORDS" "$LOCATION" --count $batch_count --start $offset)")
|
||||
echo "$result" | python -c "
|
||||
import json, sys
|
||||
data = json.loads(sys.stdin.read())
|
||||
for job in data.get('jobs', []):
|
||||
print(json.dumps(job))
|
||||
" >> output/jobs_basic.jsonl
|
||||
|
||||
job_ids=$(echo "$result" | python -c "import json,sys; [print(j['id']) for j in json.loads(sys.stdin.read()).get('jobs',[])]")
|
||||
for job_id in $job_ids; do
|
||||
detail=$(browser-act --session $SESSION eval "$(python scripts/job-detail.py $job_id)")
|
||||
echo "$detail" >> $OUTPUT_FILE
|
||||
sleep 1
|
||||
done
|
||||
|
||||
page_count=$(echo "$result" | python -c "import json,sys; print(json.loads(sys.stdin.read()).get('count',0))")
|
||||
[ "$page_count" -eq 0 ] && break
|
||||
collected=$((collected + page_count))
|
||||
offset=$((offset + page_count))
|
||||
sleep 2
|
||||
done
|
||||
echo "Done. Collected $collected jobs."
|
||||
```
|
||||
|
||||
Note: Add `sleep 1` between detail calls to avoid rate limiting. For large batches (>200 jobs), use multiple browser sessions in parallel — each session counts independently toward rate limits.
|
||||
|
||||
## Enum Parameters
|
||||
|
||||
Filter values are hardcoded in scripts; no dynamic enumeration needed.
|
||||
|
||||
Work type (`--work-type`): `1`=On-site, `2`=Remote, `3`=Hybrid
|
||||
|
||||
Contract type (`--job-type`): `F`=Full-time, `P`=Part-time, `C`=Contract, `T`=Temporary, `I`=Internship, `V`=Volunteer
|
||||
|
||||
Experience level (`--experience`): `1`=Internship, `2`=Entry level, `3`=Associate, `4`=Mid-Senior level, `5`=Director
|
||||
|
||||
Time posted (`--time-posted`): `r86400`=Past 24 hours, `r604800`=Past week, `r2592000`=Past month
|
||||
|
||||
## Pagination
|
||||
|
||||
**API Pagination**: parameter `--start`, type: page-offset, start value: `0`. Next page: increment by `--count` value. Termination: when `count` in response is `0`, or `start >= total`, or `start >= rows` target.
|
||||
|
||||
LinkedIn typically returns results up to `start=1000` maximum regardless of `total`.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
`result count >= 1` and `jobs[0].id` is non-null
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- LinkedIn limits accessible search results to approximately the first 1000 jobs per query even when `total` shows a higher number
|
||||
- `experienceLevel` may be null for many postings — companies do not always fill in this field
|
||||
- `salary` is null for most postings; LinkedIn only shows salary when the employer explicitly provides it
|
||||
- Rate limiting: sustained rapid requests (e.g., >100 detail calls without sleep) may trigger temporary blocks. Add `sleep 1` between detail calls
|
||||
- Login required: unlike public job boards, LinkedIn's Voyager API requires an authenticated session. The CSRF token is derived from the `JSESSIONID` cookie set at login
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: write a bash loop iterating over job IDs serially; do not parallelize within one browser. For higher throughput, use multiple stealth browsers with separate sessions
|
||||
- **Test before batch**: run with `--count 3` first to confirm the script runs correctly before scaling up
|
||||
- **Error resumption**: append results to `.jsonl` file line-by-line so the job can resume from a specific offset on failure
|
||||
- **Search only for large volumes**: for >500 jobs where full description is not needed, use the search component alone — it returns title, company, location, work type, and URLs without per-job detail calls
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `{working-directory}/browser-act-skill-forge-memories/linkedin-job-search-linkedin-jobs-search.memory.md` (working directory is determined by the Agent running the Skill, typically the project root or current working directory)
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file. Do not record what keywords were used or how many results were returned — those are task outputs, not experience.
|
||||
@@ -0,0 +1,66 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('job_id') # LinkedIn job posting ID (numeric)
|
||||
args = parser.parse_args()
|
||||
|
||||
js = f"""
|
||||
(async function() {{
|
||||
try {{
|
||||
const csrfToken = document.cookie.match(/JSESSIONID="([^"]+)"/)?.[1] || '';
|
||||
const headers = {{
|
||||
'csrf-token': csrfToken,
|
||||
'x-restli-protocol-version': '2.0.0',
|
||||
'accept': 'application/vnd.linkedin.normalized+json+2.1'
|
||||
}};
|
||||
const url = '/voyager/api/jobs/jobPostings/{args.job_id}?decorationId=com.linkedin.voyager.deco.jobs.web.shared.WebFullJobPosting-65';
|
||||
const res = await fetch(url, {{headers}});
|
||||
if (!res.ok) return JSON.stringify({{error: true, message: `HTTP ${{res.status}}`}});
|
||||
const data = await res.json();
|
||||
const d = data.data;
|
||||
if (!d) return JSON.stringify({{error: true, message: 'No job data returned'}});
|
||||
const workplaceMap = {{
|
||||
'urn:li:fs_workplaceType:1': 'On-site',
|
||||
'urn:li:fs_workplaceType:2': 'Remote',
|
||||
'urn:li:fs_workplaceType:3': 'Hybrid'
|
||||
}};
|
||||
const workType = d.workplaceTypes?.map(t => workplaceMap[t] || t).join(', ') || null;
|
||||
const listedAtMs = d.listedAt;
|
||||
const listedAt = listedAtMs ? new Date(listedAtMs).toISOString() : null;
|
||||
const inc = data.included || [];
|
||||
const companyUrn = d.companyDetails?.company;
|
||||
const company = inc.find(e => e.entityUrn === companyUrn);
|
||||
const companyName = company?.name || null;
|
||||
const companyUrl = company?.universalName
|
||||
? 'https://www.linkedin.com/company/' + company.universalName
|
||||
: null;
|
||||
const salary = d.salaryInsights?.insightExists
|
||||
? {{ text: d.salaryInsights?.salaryText, url: d.salaryInsights?.salaryExplorerUrl }}
|
||||
: null;
|
||||
return JSON.stringify({{
|
||||
id: '{args.job_id}',
|
||||
title: d.title,
|
||||
company: companyName,
|
||||
companyUrl,
|
||||
location: d.formattedLocation || null,
|
||||
workType,
|
||||
contractType: d.formattedEmploymentStatus || null,
|
||||
experienceLevel: d.formattedExperienceLevel || null,
|
||||
listedAt,
|
||||
applicantCount: d.applies ?? null,
|
||||
description: d.description?.text || null,
|
||||
salary,
|
||||
jobUrl: 'https://www.linkedin.com/jobs/view/{args.job_id}'
|
||||
}});
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{error: true, message: e.message}});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,82 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('keywords') # job title / keywords
|
||||
parser.add_argument('location') # location name, e.g. "United States"
|
||||
parser.add_argument('--start', default='0') # pagination offset
|
||||
parser.add_argument('--count', default='25') # results per page (max 100)
|
||||
parser.add_argument('--work-type', default='') # 1=On-site, 2=Remote, 3=Hybrid
|
||||
parser.add_argument('--job-type', default='') # F=Full-time, P=Part-time, C=Contract, T=Temporary, I=Internship, V=Volunteer
|
||||
parser.add_argument('--experience', default='') # 1=Internship, 2=Entry, 3=Associate, 4=Mid-Senior, 5=Director
|
||||
parser.add_argument('--time-posted', default='') # r86400=24h, r604800=7d, r2592000=30d
|
||||
parser.add_argument('--company-ids', default='') # comma-separated LinkedIn company IDs
|
||||
args = parser.parse_args()
|
||||
|
||||
filters = []
|
||||
if args.work_type:
|
||||
filters.append(f'workplaceType:List({args.work_type})')
|
||||
if args.job_type:
|
||||
filters.append(f'jobType:List({args.job_type})')
|
||||
if args.experience:
|
||||
filters.append(f'experience:List({args.experience})')
|
||||
if args.time_posted:
|
||||
filters.append(f'timePostedRange:List({args.time_posted})')
|
||||
if args.company_ids:
|
||||
ids = ','.join(args.company_ids.split(','))
|
||||
filters.append(f'company:List({ids})')
|
||||
|
||||
selected_filters = ('selectedFilters:(' + ','.join(filters) + '),' if filters else '')
|
||||
|
||||
js = f"""
|
||||
(async function() {{
|
||||
try {{
|
||||
const csrfToken = document.cookie.match(/JSESSIONID="([^"]+)"/)?.[1] || '';
|
||||
const headers = {{
|
||||
'csrf-token': csrfToken,
|
||||
'x-restli-protocol-version': '2.0.0',
|
||||
'accept': 'application/vnd.linkedin.normalized+json+2.1'
|
||||
}};
|
||||
const keywords = encodeURIComponent('{args.keywords}');
|
||||
const location = encodeURIComponent('{args.location}');
|
||||
const query = `(origin:JOB_SEARCH_PAGE_KEYWORD_HISTORY,keywords:${{keywords}},locationUnion:(seoLocation:(location:${{location}})),{selected_filters}spellCorrectionEnabled:true)`;
|
||||
const url = `/voyager/api/voyagerJobsDashJobCards?decorationId=com.linkedin.voyager.dash.deco.jobs.search.JobSearchCardsCollectionLite-88&count={args.count}&q=jobSearch&query=${{query}}&servedEventEnabled=false&start={args.start}`;
|
||||
const res = await fetch(url, {{headers}});
|
||||
if (!res.ok) return JSON.stringify({{error: true, message: `HTTP ${{res.status}}`}});
|
||||
const data = await res.json();
|
||||
const cards = (data.included || []).filter(e =>
|
||||
e['$type'] === 'com.linkedin.voyager.dash.jobs.JobPostingCard' &&
|
||||
e.entityUrn?.includes('JOBS_SEARCH') &&
|
||||
e.jobPostingTitle
|
||||
);
|
||||
const jobs = cards.map(c => {{
|
||||
const id = c.jobPostingUrn?.split(':').pop();
|
||||
const locRaw = c.secondaryDescription?.text || '';
|
||||
const workTypeMatch = locRaw.match(/\\(([^)]+)\\)$/);
|
||||
return {{
|
||||
id,
|
||||
title: c.jobPostingTitle,
|
||||
company: c.primaryDescription?.text || null,
|
||||
location: locRaw.replace(/\\s*\\([^)]+\\)$/, '').trim() || null,
|
||||
workType: workTypeMatch ? workTypeMatch[1] : null,
|
||||
jobUrl: id ? 'https://www.linkedin.com/jobs/view/' + id : null,
|
||||
companyUrl: c.logo?.actionTarget ? c.logo.actionTarget.replace('/life', '') : null
|
||||
}};
|
||||
}});
|
||||
return JSON.stringify({{
|
||||
total: data.data?.paging?.total || 0,
|
||||
start: {args.start},
|
||||
count: jobs.length,
|
||||
jobs
|
||||
}});
|
||||
}} catch(e) {{
|
||||
return JSON.stringify({{error: true, message: e.message}});
|
||||
}}
|
||||
}})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,220 @@
|
||||
---
|
||||
name: producthunt-launches
|
||||
description: "Scrape Product Hunt daily/weekly/monthly/yearly leaderboard launches with full product details, maker profiles, and website contact info. Use when user mentions Product Hunt, producthunt, PH scraper, product hunt launches, product hunt leaderboard, scrape product hunt, product hunt data, PH daily launches, product hunt upvotes, product hunt maker info, extract product hunt, product hunt today, top products product hunt, product hunt archive, PH products, product hunt email extraction, product hunt contact info, producthunt.com scraping, get product hunt launches, product hunt API alternative. Also applies to: startup launch monitoring, new product discovery, maker/founder contact enrichment, product hunt lead generation, daily product hunt digest, competitive product tracking."
|
||||
---
|
||||
|
||||
# Product Hunt — Launch Data Extraction
|
||||
|
||||
> Input: date/period parameters → Output: structured product launch data with maker profiles and website contact info
|
||||
|
||||
## Language
|
||||
|
||||
All process output to user (progress updates, process notifications) follows the user's language.
|
||||
|
||||
## Objective
|
||||
|
||||
Extract complete product launch data from Product Hunt leaderboard pages, enriched with maker profile information and product website contact details.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Browser session is open and can access producthunt.com
|
||||
- Cloudflare challenge may appear on first visit; use `solve-captcha` or wait for auto-pass
|
||||
|
||||
## Pre-execution Checks
|
||||
|
||||
### 1. Tool Readiness
|
||||
|
||||
If browser-act has been confirmed available in the current session → skip this step.
|
||||
|
||||
Invoke `browser-act` via Skill tool to load usage. If installation or configuration issues arise, follow its guidance to resolve then retry.
|
||||
|
||||
### 2. Cloudflare Verification
|
||||
|
||||
Product Hunt uses Cloudflare protection. On first navigation:
|
||||
1. Navigate to target URL
|
||||
2. If page title shows "Just a moment..." → `wait stable --timeout 15000` then check title again
|
||||
3. If still blocked → `solve-captcha`
|
||||
4. Verify page loaded: title should contain "Product Hunt" or "Best of Product Hunt"
|
||||
|
||||
## Capability Components
|
||||
|
||||
> This Skill's operational boundary = what the user can manually do in their browser. It only reads data already displayed to the user on the page, never bypassing authentication or access controls. JS code is encapsulated in Python files under the `scripts/` directory, invoked via `eval "$(python scripts/xxx.py)"`. `$(...)` is bash syntax; use the bash tool for execution.
|
||||
|
||||
### DOM: Extract product list from leaderboard page
|
||||
|
||||
Navigate to the target leaderboard URL first, then extract:
|
||||
|
||||
`eval "$(python scripts/extract-leaderboard.py)"`
|
||||
|
||||
URL patterns (navigate to the appropriate one before extraction):
|
||||
- Daily: `https://www.producthunt.com/leaderboard/daily/{YYYY}/{M}/{DD}/all`
|
||||
- Weekly: `https://www.producthunt.com/leaderboard/weekly/{YYYY}/{week-number}/all`
|
||||
- Monthly: `https://www.producthunt.com/leaderboard/monthly/{YYYY}/{M}/all`
|
||||
- Yearly: `https://www.producthunt.com/leaderboard/yearly/{YYYY}/all`
|
||||
|
||||
Replace `all` with `featured` for featured-only products.
|
||||
|
||||
Output example:
|
||||
```json
|
||||
[
|
||||
{
|
||||
"rank": 1,
|
||||
"name": "Product Name",
|
||||
"tagline": "Short product description",
|
||||
"categories": ["Productivity", "AI"],
|
||||
"thumbnail": "https://ph-files.imgix.net/...",
|
||||
"upvotes": 135,
|
||||
"comments": 42,
|
||||
"url": "https://www.producthunt.com/products/product-slug",
|
||||
"slug": "product-slug"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### DOM: Extract launch detail page
|
||||
|
||||
Navigate to the launch page URL first (`https://www.producthunt.com/products/{slug}/launches/{launch-slug}`), then extract:
|
||||
|
||||
`eval "$(python scripts/extract-launch-detail.py)"`
|
||||
|
||||
To find the launch URL from a product page: navigate to `https://www.producthunt.com/products/{slug}` and look for links matching `/products/{slug}/launches/{launch-slug}`.
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"name": "Product Name",
|
||||
"tagline": "Short product tagline",
|
||||
"description": "Full product description from OG meta",
|
||||
"categories": ["Productivity", "Social Media"],
|
||||
"images": ["https://ph-files.imgix.net/gallery1.png", "https://ph-files.imgix.net/gallery2.png"],
|
||||
"websiteUrl": "https://product-website.com/?ref=producthunt",
|
||||
"upvotes": 135,
|
||||
"launchDate": "2025-05-27T07:26:33-07:00",
|
||||
"makers": [{"href": "/@username", "name": "Maker Name"}],
|
||||
"ogImage": "https://ph-files.imgix.net/og-image.png"
|
||||
}
|
||||
```
|
||||
|
||||
### DOM: Extract maker profile
|
||||
|
||||
Navigate to maker profile URL (`https://www.producthunt.com/@{username}`), then extract:
|
||||
|
||||
`eval "$(python scripts/extract-maker-profile.py)"`
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"name": "Maker Name",
|
||||
"slug": "@username",
|
||||
"headline": "Creating SaaS Products",
|
||||
"aboutText": "Bio text about the maker",
|
||||
"links": ["https://twitter.com/username", "https://linkedin.com/in/username"],
|
||||
"followers": 22,
|
||||
"url": "https://www.producthunt.com/@username"
|
||||
}
|
||||
```
|
||||
|
||||
### DOM: Extract website email and content
|
||||
|
||||
Navigate to the product website URL, wait for load, then extract:
|
||||
|
||||
`eval "$(python scripts/extract-website-content.py)"`
|
||||
|
||||
Alternatively, use `stealth-extract` for faster extraction without a browser session:
|
||||
`stealth-extract {website-url} --content-type markdown` then parse the markdown for email patterns.
|
||||
|
||||
Output example:
|
||||
```json
|
||||
{
|
||||
"title": "Product Website Title",
|
||||
"url": "https://product-website.com",
|
||||
"email": "contact@product-website.com",
|
||||
"allEmails": ["contact@product-website.com", "support@product-website.com"],
|
||||
"websiteRawText": "Full visible text content of the website..."
|
||||
}
|
||||
```
|
||||
|
||||
### Composite: Full product extraction (leaderboard + detail + maker + website)
|
||||
|
||||
Complete pipeline replicating the full Product Hunt scraper workflow:
|
||||
|
||||
1. Navigate to leaderboard page → `wait stable` → `eval "$(python scripts/extract-leaderboard.py)"`
|
||||
2. For each product from step 1:
|
||||
a. Navigate to `https://www.producthunt.com/products/{slug}` → find launch link → navigate to launch page
|
||||
b. `wait stable` → `eval "$(python scripts/extract-launch-detail.py)"` → get full details + maker links + website URL
|
||||
3. (Optional, if `scrapeMakers` is enabled) For each unique maker from step 2:
|
||||
a. Navigate to `https://www.producthunt.com/{maker.href}` → `wait stable` → `eval "$(python scripts/extract-maker-profile.py)"`
|
||||
4. (Optional, if `scrapeWebsite` is enabled) For each product website URL from step 2:
|
||||
a. Navigate to website URL → `wait stable` → `eval "$(python scripts/extract-website-content.py)"`
|
||||
5. Merge all data by product slug
|
||||
|
||||
Final output example per product:
|
||||
```json
|
||||
{
|
||||
"date": "2026-06-10T00:00:00Z",
|
||||
"launchDate": "2026-06-10T07:01:04Z",
|
||||
"url": "https://www.producthunt.com/products/product-slug",
|
||||
"name": "Product Name",
|
||||
"shortDescription": "Short tagline",
|
||||
"description": "Full description text",
|
||||
"categories": ["Productivity", "AI"],
|
||||
"maker": {
|
||||
"makerHref": "https://www.producthunt.com/@username",
|
||||
"name": "Maker Name",
|
||||
"slug": "@username",
|
||||
"url": "https://www.producthunt.com/@username",
|
||||
"links": ["https://twitter.com/maker", "https://linkedin.com/in/maker"],
|
||||
"aboutText": "Maker bio text"
|
||||
},
|
||||
"websiteUrl": "https://product-website.com",
|
||||
"images": ["https://ph-files.imgix.net/image1.png"],
|
||||
"upvotes": 135,
|
||||
"website": {
|
||||
"title": "Product Website",
|
||||
"url": "https://product-website.com",
|
||||
"email": "hello@product-website.com",
|
||||
"websiteRawText": "Full page text content..."
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Pagination
|
||||
|
||||
**No pagination required for daily/weekly leaderboard**: All products for a given day load on a single page (typically 15-50 products per day). No infinite scroll or "load more" button exists.
|
||||
|
||||
**Yearly leaderboard**: May contain many products. Apply `topNProducts` filter to limit. All visible products are rendered on the single page.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- `result count >= 1` (at least one product extracted from leaderboard)
|
||||
- Core fields non-null: `name`, `tagline`, `upvotes`, `url` present for every product
|
||||
- Data consistency: extracted product names match what is displayed on the page
|
||||
- When detail enrichment is performed: `websiteUrl` or `maker` present for enriched items
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Cloudflare protection requires initial challenge pass; may need `solve-captcha` on first visit
|
||||
- No public API available; Product Hunt only accepts persisted GraphQL queries. All data must be extracted via DOM
|
||||
- Rate limiting: rapid sequential page navigations may trigger Cloudflare blocks. Add 2-3 second delays between product detail page visits
|
||||
- The `/all` URL path (used by older scrapers) now returns 404; use `/leaderboard/daily/` path instead
|
||||
- Product detail pages may vary in structure for older launches vs newer ones
|
||||
- Website email extraction depends on email being visible in page text or HTML; mailto links and contact forms with obfuscated emails will not be captured
|
||||
|
||||
## Execution Efficiency
|
||||
|
||||
- **Batch orchestration**: Write a bash script to loop through the command templates serially within a single session; do not parallelize within one browser. Add 2-3 second delays between navigations to avoid Cloudflare blocks. To increase throughput, open multiple stealth browser sessions and distribute work across them
|
||||
- **Test before batch execution**: After writing a batch script, first test with 1-2 items to verify the script runs correctly; only then run the full batch
|
||||
- **Reduce redundant pre-operations**: The leaderboard extraction gives all basic data in one pass; only visit detail pages when full description, images, or maker info are needed
|
||||
- **Error resumption**: Save results item by item during batch processing; on failure, resume from the breakpoint rather than starting over
|
||||
- **Skip website extraction when not needed**: Website content extraction is the slowest step (external site navigation). Only enable when email/content data is specifically required
|
||||
|
||||
## Experience Notes
|
||||
|
||||
Path: `browser-act-skill-forge-memories/producthunt-scraper-producthunt-launches.memory.md` (working directory is determined by the Agent running the Skill)
|
||||
|
||||
**Before execution**: If the file exists, read it first — it records unexpected situations encountered during past executions (e.g., a strategy has become ineffective); adjust strategy order accordingly.
|
||||
|
||||
**After execution**: If an unexpected situation is encountered (strategy became ineffective, page redesigned, anti-scraping upgraded, better path discovered), append a line:
|
||||
`{YYYY-MM-DD}: {what happened} → {conclusion}`
|
||||
|
||||
Normal execution does not write to the file.
|
||||
@@ -0,0 +1,78 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
args = parser.parse_args()
|
||||
|
||||
js = """
|
||||
(()=>{
|
||||
try {
|
||||
const title = document.querySelector('h1')?.textContent?.trim() || '';
|
||||
const ogDesc = document.querySelector('meta[property="og:description"]')?.content || '';
|
||||
const ogImage = document.querySelector('meta[property="og:image"]')?.content || '';
|
||||
|
||||
const allLinks = Array.from(document.querySelectorAll('a[href]'));
|
||||
const visitLink = allLinks.find(a => {
|
||||
const txt = a.textContent.trim().toLowerCase();
|
||||
return txt.includes('visit') && a.href && !a.href.includes('producthunt.com');
|
||||
});
|
||||
const websiteUrl = visitLink ? visitLink.href : null;
|
||||
|
||||
const topicLinks = Array.from(document.querySelectorAll('a[href^="/topics/"]'));
|
||||
const categories = [...new Set(topicLinks.map(a => a.textContent.trim()))];
|
||||
|
||||
const imgs = Array.from(document.querySelectorAll('img[src*="ph-files.imgix.net"]'));
|
||||
const images = imgs.map(i => i.src).filter(s => s.includes('fit=max'));
|
||||
|
||||
const upvoteBtn = Array.from(document.querySelectorAll('button')).find(b => /upvote\\s*\\d+/i.test(b.textContent));
|
||||
const upvotes = upvoteBtn ? parseInt(upvoteBtn.textContent.match(/\\d+/)?.[0]) || 0 : 0;
|
||||
|
||||
const timeEls = Array.from(document.querySelectorAll('time'));
|
||||
const launchDate = timeEls.length > 0 ? timeEls[0].getAttribute('datetime') : null;
|
||||
|
||||
const makerLinks = allLinks.filter(a => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
return href.match(/^\\/@[a-z0-9_]+$/i);
|
||||
}).map(a => ({href: a.getAttribute('href'), name: a.textContent.trim()}))
|
||||
.filter(m => m.name.length > 0);
|
||||
const uniqueMakers = [...new Map(makerLinks.map(m => [m.href, m])).values()];
|
||||
|
||||
const description = ogDesc;
|
||||
|
||||
const bodyText = document.body.innerText;
|
||||
let tagline = '';
|
||||
const lines = bodyText.split('\\n').filter(l => l.trim());
|
||||
const visitLineIdx = lines.findIndex(l => l.trim() === 'Visit');
|
||||
if (visitLineIdx > 1) {
|
||||
tagline = lines[visitLineIdx - 1].trim();
|
||||
}
|
||||
if (!tagline || tagline === title) {
|
||||
const ogTitle = document.querySelector('meta[property="og:title"]')?.content || '';
|
||||
if (ogTitle && ogTitle !== title) tagline = ogTitle;
|
||||
}
|
||||
|
||||
return JSON.stringify({
|
||||
name: title,
|
||||
tagline: tagline,
|
||||
description: description,
|
||||
categories: categories,
|
||||
images: images,
|
||||
websiteUrl: websiteUrl,
|
||||
upvotes: upvotes,
|
||||
launchDate: launchDate,
|
||||
makers: uniqueMakers,
|
||||
ogImage: ogImage
|
||||
});
|
||||
} catch(e) {
|
||||
return JSON.stringify({error: true, message: e.message});
|
||||
}
|
||||
})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,49 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--date', default='', help='Date in YYYY-M-DD format for daily archive')
|
||||
parser.add_argument('--year', default='', help='Year in YYYY format for yearly archive')
|
||||
parser.add_argument('--week', default='', help='Week number for weekly archive')
|
||||
parser.add_argument('--month', default='', help='Month number for monthly archive')
|
||||
parser.add_argument('--filter-type', default='all', help='Filter type: all or featured')
|
||||
args = parser.parse_args()
|
||||
|
||||
js = """
|
||||
(()=>{
|
||||
try {
|
||||
const cards = Array.from(document.querySelectorAll('section.group'));
|
||||
if(cards.length === 0) return JSON.stringify({error: true, message: "No product cards found on page. Check if the page loaded correctly."});
|
||||
const products = cards.map(card => {
|
||||
const link = card.querySelector('a[href^="/products/"]');
|
||||
if(!link) return null;
|
||||
const linkText = link.textContent.trim();
|
||||
const rankMatch = linkText.match(/^(\\d+)\\.\\s+(.+)/);
|
||||
const rank = rankMatch ? parseInt(rankMatch[1]) : null;
|
||||
const name = rankMatch ? rankMatch[2] : linkText;
|
||||
const topicLinks = Array.from(card.querySelectorAll('a[href^="/topics/"]'));
|
||||
const categories = topicLinks.map(a => a.textContent.trim());
|
||||
const img = card.querySelector('img[alt]:not([alt="Promoted"])');
|
||||
const thumbnail = img ? img.src : null;
|
||||
const lines = card.innerText.split('\\n').filter(l => l.trim());
|
||||
const tagline = lines.length > 1 ? lines[1] : '';
|
||||
const buttons = Array.from(card.querySelectorAll('button.relative'));
|
||||
const upvotes = buttons[0] ? parseInt(buttons[0].textContent.trim()) || 0 : 0;
|
||||
const comments = buttons[1] ? parseInt(buttons[1].textContent.trim()) || 0 : 0;
|
||||
const href = link.getAttribute('href');
|
||||
return { rank, name, tagline, categories, thumbnail, upvotes, comments, url: 'https://www.producthunt.com' + href, slug: href.replace('/products/', '') };
|
||||
}).filter(Boolean);
|
||||
return JSON.stringify(products);
|
||||
} catch(e) {
|
||||
return JSON.stringify({error: true, message: e.message});
|
||||
}
|
||||
})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,53 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
args = parser.parse_args()
|
||||
|
||||
js = """
|
||||
(()=>{
|
||||
try {
|
||||
const name = document.querySelector('h1')?.textContent?.trim() || '';
|
||||
const bodyText = document.body.innerText;
|
||||
|
||||
const headlineMatch = bodyText.match(new RegExp(name + '\\n(.+?)\\n'));
|
||||
const headline = headlineMatch ? headlineMatch[1].trim() : '';
|
||||
|
||||
const aboutSection = bodyText.match(/About\\n([\\s\\S]*?)(?=Links|Badges|Maker History|Forums|$)/);
|
||||
const aboutText = aboutSection ? aboutSection[1].trim().split('\\n').filter(l => l.trim()).join(' ') : '';
|
||||
|
||||
const allLinks = Array.from(document.querySelectorAll('a[href]'));
|
||||
const externalLinks = allLinks.filter(a => {
|
||||
const href = a.href;
|
||||
return href && !href.includes('producthunt.com') && !href.includes('javascript:') && !href.includes('#') && !href.includes('google.com') && !href.includes('lu.ma');
|
||||
}).map(a => a.href);
|
||||
const uniqueLinks = [...new Set(externalLinks)];
|
||||
|
||||
const followersMatch = bodyText.match(/(\\d+)\\s*followers/i);
|
||||
const followers = followersMatch ? parseInt(followersMatch[1]) : 0;
|
||||
|
||||
const slugMatch = location.pathname.match(/^\\/@(.+)/);
|
||||
const slug = slugMatch ? '@' + slugMatch[1] : '';
|
||||
|
||||
return JSON.stringify({
|
||||
name: name,
|
||||
slug: slug,
|
||||
headline: headline,
|
||||
aboutText: aboutText,
|
||||
links: uniqueLinks,
|
||||
followers: followers,
|
||||
url: location.href
|
||||
});
|
||||
} catch(e) {
|
||||
return JSON.stringify({error: true, message: e.message});
|
||||
}
|
||||
})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||
parser = argparse.ArgumentParser()
|
||||
args = parser.parse_args()
|
||||
|
||||
js = """
|
||||
(()=>{
|
||||
try {
|
||||
const title = document.title || '';
|
||||
const url = location.href;
|
||||
const bodyText = document.body.innerText || '';
|
||||
|
||||
const emailRegex = /[a-zA-Z0-9._%+\\-]+@[a-zA-Z0-9.\\-]+\\.[a-zA-Z]{2,}/g;
|
||||
const allEmails = bodyText.match(emailRegex) || [];
|
||||
const html = document.documentElement.innerHTML;
|
||||
const htmlEmails = html.match(emailRegex) || [];
|
||||
const combined = [...new Set([...allEmails, ...htmlEmails])];
|
||||
const filtered = combined.filter(e => !e.includes('sentry') && !e.includes('webpack') && !e.includes('example.com'));
|
||||
const email = filtered.length > 0 ? filtered[0] : null;
|
||||
|
||||
const rawText = bodyText.replace(/\\s+/g, ' ').trim().slice(0, 5000);
|
||||
|
||||
return JSON.stringify({
|
||||
title: title,
|
||||
url: url,
|
||||
email: email,
|
||||
allEmails: filtered,
|
||||
websiteRawText: rawText
|
||||
});
|
||||
} catch(e) {
|
||||
return JSON.stringify({error: true, message: e.message});
|
||||
}
|
||||
})()
|
||||
"""
|
||||
print(js)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,80 @@
|
||||
---
|
||||
name: social-media-finder-skill
|
||||
description: "This skill helps users automatically find social media profiles across platforms like Facebook, Twitter, Instagram, LinkedIn, etc. using the BrowserAct API. Agent should proactively apply this skill when users express needs like finding someone's social media accounts, discovering a brand's social media presence, tracking down social profiles of job candidates, finding contact info for sales prospects, researching a person's digital footprint, verifying the identity of someone met online, monitoring the social media reach of influencers, checking social accounts of business competitors, gathering public profiles for background research, locating official customer service accounts across platforms, or compiling contact databases for networking."
|
||||
metadata: {"openclaw":{"emoji":"🌐","requires":{"bins":["python"],"env":["BROWSERACT_API_KEY"]}}}
|
||||
---
|
||||
|
||||
# Social Media Finder Skill
|
||||
|
||||
## 📖 Brief
|
||||
This skill automates the discovery of social media accounts associated with individuals, brands, or businesses across multiple platforms. By providing a name, it searches across Facebook, Twitter, Instagram, LinkedIn, TikTok, and more — extracting profile URLs, usernames, follower counts, and bio snippets in one go. The results are returned as a downloadable CSV file.
|
||||
|
||||
## ✨ Features
|
||||
1. **No Hallucinations**: Pre-set workflows avoid AI generative hallucinations, ensuring stable and precise data extraction.
|
||||
2. **No Captcha Issues**: No need to handle reCAPTCHA or other verification challenges.
|
||||
3. **No IP Restrictions**: No need to handle regional IP restrictions or geofencing.
|
||||
4. **Faster Execution**: Tasks execute faster compared to pure AI-driven browser automation solutions.
|
||||
5. **Cost-Effective**: Significantly lowers data acquisition costs compared to high-token-consuming AI solutions.
|
||||
|
||||
## 🔑 API Key Setup
|
||||
Before running, check the `BROWSERACT_API_KEY` environment variable. If not set, do not take other measures; ask and wait for the user to provide it.
|
||||
**Agent must inform the user**:
|
||||
> "Since you haven't configured the BrowserAct API Key yet, please visit the [BrowserAct Console](https://www.browseract.com/reception/integrations) to get your Key."
|
||||
|
||||
## 🛠️ Input Parameters
|
||||
The agent should configure the following parameter based on user requirements:
|
||||
|
||||
1. **People_Name**
|
||||
- **Required**: Yes
|
||||
- **Type**: `string`
|
||||
- **Description**: The name of the person or brand to search for. Use `+` between words for better results.
|
||||
- **Example**: `John+Smith`, `Tesla`, `Bill+Gates`
|
||||
- **Default**: `Mike+Smith`
|
||||
|
||||
## 🚀 Invocation Method
|
||||
Agent should execute the following command to invoke the skill:
|
||||
|
||||
```bash
|
||||
# Example invocation
|
||||
python -u ./scripts/social_media_finder.py "John+Smith"
|
||||
```
|
||||
|
||||
### ⏳ Execution Monitoring
|
||||
Since this task involves automated browser operations, it may take several minutes. The script outputs **timestamped status logs** continuously (e.g., `[14:30:05] Task Status: running`).
|
||||
**Agent guidelines**:
|
||||
- Monitor the terminal output while waiting.
|
||||
- As long as new status logs appear, the task is running normally; do not misjudge it as frozen.
|
||||
- Only consider triggering retry if the status remains unchanged for a long time or output stops without a final result.
|
||||
|
||||
## 📊 Data Output
|
||||
Upon successful execution, the script retrieves a JSON object containing a `files` array with a download link to a CSV file. The CSV includes the following structured profile data:
|
||||
- `Platform name`: The social media platform (e.g., LinkedIn, Twitter).
|
||||
- `Followers count`: The number of followers.
|
||||
- `Page title`: The title of the profile page.
|
||||
- `Description snippet`: The bio or description snippet.
|
||||
- `Profile URL`: The direct link to the social media profile.
|
||||
|
||||
## ⚠️ Error Handling & Retry
|
||||
If an error occurs during script execution (e.g., network fluctuations or task failure), the Agent should follow this logic:
|
||||
|
||||
1. **Check Output Content**:
|
||||
- If the output **contains** `"Invalid authorization"`, it means the API Key is invalid or expired. **Do not retry**; guide the user to re-check and provide the correct API Key.
|
||||
- If the output **contains** `"concurrent"` or `"too many running tasks"`, it means the concurrent task limit has been reached. **Do not retry**; guide the user to upgrade their plan.
|
||||
**Agent must inform the user**:
|
||||
> "The current task cannot be executed because your BrowserAct account has reached the concurrent task limit. Please visit the [BrowserAct Plan Upgrade Page](https://www.browseract.com/reception/recharge) to upgrade your plan."
|
||||
- If the output **does not contain the above error keywords** but the task failed (e.g., output starts with `Error:` or returns empty results), the Agent should **automatically re-execute the script once**.
|
||||
|
||||
2. **Retry Limit**:
|
||||
- Automatic retry is limited to **one time**. If the second attempt fails, stop retrying and report the specific error to the user.
|
||||
|
||||
## 🌟 Typical Use Cases
|
||||
1. **Background Research**: Find social media profiles of job candidates or new connections.
|
||||
2. **Lead Generation**: Gather contact information and social profiles for sales prospects.
|
||||
3. **Competitive Intelligence**: Discover the social media footprint of competitors.
|
||||
4. **Influencer Discovery**: Locate content creators across various platforms.
|
||||
5. **Brand Monitoring**: Check a brand's presence across different social networks.
|
||||
6. **Reconnecting**: Find old friends or colleagues online.
|
||||
7. **Identity Verification**: Cross-reference profiles of people met online.
|
||||
8. **Customer Support**: Find official brand accounts for reaching out.
|
||||
9. **Networking**: Research event attendees or potential business partners.
|
||||
10. **Digital Footprint Audit**: Monitor one's own public social media presence.
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user