chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,124 @@
|
||||
# Crawl4AI Chrome Extension
|
||||
|
||||
Visual extraction tools for Crawl4AI - Click to extract data and content from any webpage!
|
||||
|
||||
## 🚀 Features
|
||||
|
||||
- **Click2Crawl**: Click on elements to build data extraction schemas instantly
|
||||
- **Markdown Extraction**: Select elements and export as clean markdown
|
||||
- **Script Builder (Alpha)**: Record browser actions to create automation scripts
|
||||
- **Smart Element Selection**: Container and field selection with visual feedback
|
||||
- **Code Generation**: Generates complete Python code for Crawl4AI
|
||||
- **Beautiful Dark UI**: Consistent with Crawl4AI's design language
|
||||
|
||||
## 📦 Installation
|
||||
|
||||
### Method 1: Load Unpacked Extension (Recommended for Development)
|
||||
|
||||
1. Open Chrome and navigate to `chrome://extensions/`
|
||||
2. Enable "Developer mode" in the top right corner
|
||||
3. Click "Load unpacked"
|
||||
4. Select the `crawl4ai-assistant` folder
|
||||
5. The extension icon (🚀🤖) will appear in your toolbar
|
||||
|
||||
### Method 2: Generate Icons First
|
||||
|
||||
If you want proper icons:
|
||||
|
||||
1. Open `icons/generate_icons.html` in your browser
|
||||
2. Right-click each canvas and save as:
|
||||
- `icon-16.png`
|
||||
- `icon-48.png`
|
||||
- `icon-128.png`
|
||||
3. Then follow Method 1 above
|
||||
|
||||
## 🎯 How to Use
|
||||
|
||||
### Using Click2Crawl
|
||||
|
||||
1. **Navigate to any website** you want to extract data from
|
||||
2. **Click the Crawl4AI extension icon** in your toolbar
|
||||
3. **Click "Click2Crawl"** to start the capture mode
|
||||
4. **Select a container element**:
|
||||
- Hover over elements (they'll highlight in blue)
|
||||
- Click on a repeating container (e.g., product card, article block)
|
||||
5. **Select fields within the container**:
|
||||
- Elements will now highlight in green
|
||||
- Click on each piece of data you want to extract
|
||||
- Name each field (e.g., "title", "price", "description")
|
||||
6. **Test and Export**:
|
||||
- Click "Test Schema" to see extracted data instantly
|
||||
- Export as Python code, JSON schema, or markdown
|
||||
|
||||
### Running the Generated Code
|
||||
|
||||
The downloaded Python file contains:
|
||||
|
||||
```python
|
||||
# 1. The HTML snippet of your selected container
|
||||
HTML_SNIPPET = """..."""
|
||||
|
||||
# 2. The extraction query based on your selections
|
||||
EXTRACTION_QUERY = """..."""
|
||||
|
||||
# 3. Functions to generate and test the schema
|
||||
async def generate_schema():
|
||||
# Generates the extraction schema using LLM
|
||||
|
||||
async def test_extraction():
|
||||
# Tests the schema on the actual website
|
||||
```
|
||||
|
||||
To use it:
|
||||
|
||||
1. Install Crawl4AI: `pip install crawl4ai`
|
||||
2. Run the script: `python crawl4ai_schema_*.py`
|
||||
3. The script will generate a `generated_schema.json` file
|
||||
4. Use this schema in your Crawl4AI projects!
|
||||
|
||||
## 🎨 Visual Feedback
|
||||
|
||||
- **Blue dashed outline**: Container selection mode
|
||||
- **Green dashed outline**: Field selection mode
|
||||
- **Solid blue outline**: Selected container
|
||||
- **Solid green outline**: Selected fields
|
||||
- **Floating toolbar**: Shows current mode and selection status
|
||||
|
||||
## ⌨️ Keyboard Shortcuts
|
||||
|
||||
- **ESC**: Cancel current capture session
|
||||
|
||||
## 🔧 Technical Details
|
||||
|
||||
- Built with Manifest V3 for security and performance
|
||||
- Pure client-side - no data sent to external servers
|
||||
- Generates code that uses Crawl4AI's LLM integration
|
||||
- Smart selector generation prioritizes stable attributes
|
||||
|
||||
## 🐛 Troubleshooting
|
||||
|
||||
### Extension doesn't load
|
||||
- Make sure you're in Developer Mode
|
||||
- Check the console for any errors
|
||||
- Ensure all files are in the correct directories
|
||||
|
||||
### Can't select elements
|
||||
- Some websites may block extensions
|
||||
- Try refreshing the page
|
||||
- Make sure you clicked "Schema Builder" first
|
||||
|
||||
### Generated code doesn't work
|
||||
- Ensure you have Crawl4AI installed
|
||||
- Check that you have an LLM API key configured
|
||||
- Make sure the website structure hasn't changed
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
This extension is part of the Crawl4AI project. Contributions are welcome!
|
||||
|
||||
- Report issues: [GitHub Issues](https://github.com/unclecode/crawl4ai/issues)
|
||||
- Join discussion: [Discord](https://discord.gg/crawl4ai)
|
||||
|
||||
## 📄 License
|
||||
|
||||
Same as Crawl4AI - see main project for details.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,39 @@
|
||||
// Service worker for Crawl4AI Assistant
|
||||
|
||||
// Handle messages from content script
|
||||
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
|
||||
if (message.action === 'downloadCode' || message.action === 'downloadScript') {
|
||||
try {
|
||||
// Create a data URL for the Python code
|
||||
const dataUrl = 'data:text/plain;charset=utf-8,' + encodeURIComponent(message.code);
|
||||
|
||||
// Download the file
|
||||
chrome.downloads.download({
|
||||
url: dataUrl,
|
||||
filename: message.filename || 'crawl4ai_schema.py',
|
||||
saveAs: true
|
||||
}, (downloadId) => {
|
||||
if (chrome.runtime.lastError) {
|
||||
console.error('Download failed:', chrome.runtime.lastError);
|
||||
sendResponse({ success: false, error: chrome.runtime.lastError.message });
|
||||
} else {
|
||||
console.log('Download started with ID:', downloadId);
|
||||
sendResponse({ success: true, downloadId: downloadId });
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error creating download:', error);
|
||||
sendResponse({ success: false, error: error.message });
|
||||
}
|
||||
|
||||
return true; // Keep the message channel open for async response
|
||||
}
|
||||
|
||||
return false;
|
||||
});
|
||||
|
||||
// Clean up on extension install/update
|
||||
chrome.runtime.onInstalled.addListener(() => {
|
||||
// Clear any stored state
|
||||
chrome.storage.local.clear();
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,78 @@
|
||||
// Main content script for Crawl4AI Assistant
|
||||
// Coordinates between Click2Crawl, ScriptBuilder, and MarkdownExtraction
|
||||
|
||||
let activeBuilder = null;
|
||||
|
||||
// Listen for messages from popup
|
||||
chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
|
||||
if (request.action === 'startCapture') {
|
||||
if (activeBuilder) {
|
||||
console.log('Stopping existing capture session');
|
||||
activeBuilder.stop();
|
||||
activeBuilder = null;
|
||||
}
|
||||
|
||||
if (request.mode === 'schema') {
|
||||
console.log('Starting Click2Crawl');
|
||||
activeBuilder = new Click2Crawl();
|
||||
activeBuilder.start();
|
||||
} else if (request.mode === 'script') {
|
||||
console.log('Starting Script Builder');
|
||||
activeBuilder = new ScriptBuilder();
|
||||
activeBuilder.start();
|
||||
}
|
||||
|
||||
sendResponse({ success: true });
|
||||
} else if (request.action === 'stopCapture') {
|
||||
if (activeBuilder) {
|
||||
activeBuilder.stop();
|
||||
activeBuilder = null;
|
||||
}
|
||||
sendResponse({ success: true });
|
||||
} else if (request.action === 'startSchemaCapture') {
|
||||
if (activeBuilder) {
|
||||
activeBuilder.deactivate?.();
|
||||
activeBuilder = null;
|
||||
}
|
||||
console.log('Starting Click2Crawl');
|
||||
activeBuilder = new Click2Crawl();
|
||||
activeBuilder.start();
|
||||
sendResponse({ success: true });
|
||||
} else if (request.action === 'startScriptCapture') {
|
||||
if (activeBuilder) {
|
||||
activeBuilder.deactivate?.();
|
||||
activeBuilder = null;
|
||||
}
|
||||
console.log('Starting Script Builder');
|
||||
activeBuilder = new ScriptBuilder();
|
||||
activeBuilder.start();
|
||||
sendResponse({ success: true });
|
||||
} else if (request.action === 'startClick2Crawl') {
|
||||
if (activeBuilder) {
|
||||
activeBuilder.deactivate?.();
|
||||
activeBuilder = null;
|
||||
}
|
||||
console.log('Starting Markdown Extraction');
|
||||
activeBuilder = new MarkdownExtraction();
|
||||
sendResponse({ success: true });
|
||||
} else if (request.action === 'generateCode') {
|
||||
if (activeBuilder && activeBuilder.generateCode) {
|
||||
activeBuilder.generateCode();
|
||||
}
|
||||
sendResponse({ success: true });
|
||||
}
|
||||
});
|
||||
|
||||
// Cleanup on page unload
|
||||
window.addEventListener('beforeunload', () => {
|
||||
if (activeBuilder) {
|
||||
if (activeBuilder.deactivate) {
|
||||
activeBuilder.deactivate();
|
||||
} else if (activeBuilder.stop) {
|
||||
activeBuilder.stop();
|
||||
}
|
||||
activeBuilder = null;
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Crawl4AI Assistant content script loaded');
|
||||
@@ -0,0 +1,623 @@
|
||||
class ContentAnalyzer {
|
||||
constructor() {
|
||||
this.patterns = {
|
||||
article: ['article', 'main', 'content', 'post', 'entry'],
|
||||
navigation: ['nav', 'menu', 'navigation', 'breadcrumb'],
|
||||
sidebar: ['sidebar', 'aside', 'widget'],
|
||||
header: ['header', 'masthead', 'banner'],
|
||||
footer: ['footer', 'copyright', 'contact'],
|
||||
list: ['list', 'items', 'results', 'products', 'cards'],
|
||||
table: ['table', 'grid', 'data'],
|
||||
media: ['gallery', 'carousel', 'slideshow', 'video', 'media']
|
||||
};
|
||||
}
|
||||
|
||||
async analyze(elements) {
|
||||
const analysis = {
|
||||
structure: await this.analyzeStructure(elements),
|
||||
contentType: this.identifyContentType(elements),
|
||||
hierarchy: this.buildHierarchy(elements),
|
||||
mediaAssets: this.collectMediaAssets(elements),
|
||||
textDensity: this.calculateTextDensity(elements),
|
||||
semanticRegions: this.identifySemanticRegions(elements),
|
||||
relationships: this.analyzeRelationships(elements),
|
||||
metadata: this.extractMetadata(elements)
|
||||
};
|
||||
|
||||
return analysis;
|
||||
}
|
||||
|
||||
analyzeStructure(elements) {
|
||||
const structure = {
|
||||
hasHeadings: false,
|
||||
hasLists: false,
|
||||
hasTables: false,
|
||||
hasMedia: false,
|
||||
hasCode: false,
|
||||
hasLinks: false,
|
||||
layout: 'linear', // linear, grid, mixed
|
||||
depth: 0,
|
||||
elementTypes: new Map()
|
||||
};
|
||||
|
||||
// Analyze each element
|
||||
for (const element of elements) {
|
||||
this.analyzeElementStructure(element, structure);
|
||||
}
|
||||
|
||||
// Determine layout type
|
||||
structure.layout = this.determineLayout(elements);
|
||||
|
||||
// Calculate max depth
|
||||
structure.depth = this.calculateMaxDepth(elements);
|
||||
|
||||
return structure;
|
||||
}
|
||||
|
||||
analyzeElementStructure(element, structure, visited = new Set()) {
|
||||
if (visited.has(element)) return;
|
||||
visited.add(element);
|
||||
|
||||
const tagName = element.tagName;
|
||||
|
||||
// Update element type count
|
||||
structure.elementTypes.set(
|
||||
tagName,
|
||||
(structure.elementTypes.get(tagName) || 0) + 1
|
||||
);
|
||||
|
||||
// Check for specific structures
|
||||
if (/^H[1-6]$/.test(tagName)) {
|
||||
structure.hasHeadings = true;
|
||||
} else if (['UL', 'OL', 'DL'].includes(tagName)) {
|
||||
structure.hasLists = true;
|
||||
} else if (tagName === 'TABLE') {
|
||||
structure.hasTables = true;
|
||||
} else if (['IMG', 'VIDEO', 'IFRAME', 'PICTURE'].includes(tagName)) {
|
||||
structure.hasMedia = true;
|
||||
} else if (['CODE', 'PRE'].includes(tagName)) {
|
||||
structure.hasCode = true;
|
||||
} else if (tagName === 'A') {
|
||||
structure.hasLinks = true;
|
||||
}
|
||||
|
||||
// Analyze children
|
||||
for (const child of element.children) {
|
||||
this.analyzeElementStructure(child, structure, visited);
|
||||
}
|
||||
}
|
||||
|
||||
identifyContentType(elements) {
|
||||
const scores = {
|
||||
article: 0,
|
||||
list: 0,
|
||||
table: 0,
|
||||
form: 0,
|
||||
media: 0,
|
||||
mixed: 0
|
||||
};
|
||||
|
||||
for (const element of elements) {
|
||||
// Score based on element types and classes
|
||||
const tagName = element.tagName;
|
||||
const className = element.className.toLowerCase();
|
||||
const id = element.id.toLowerCase();
|
||||
|
||||
// Check for article patterns
|
||||
if (tagName === 'ARTICLE' ||
|
||||
this.matchesPattern(className + ' ' + id, this.patterns.article)) {
|
||||
scores.article += 10;
|
||||
}
|
||||
|
||||
// Check for list patterns
|
||||
if (['UL', 'OL'].includes(tagName) ||
|
||||
this.matchesPattern(className, this.patterns.list)) {
|
||||
scores.list += 5;
|
||||
}
|
||||
|
||||
// Check for table
|
||||
if (tagName === 'TABLE') {
|
||||
scores.table += 10;
|
||||
}
|
||||
|
||||
// Check for form
|
||||
if (tagName === 'FORM' || element.querySelector('input, select, textarea')) {
|
||||
scores.form += 5;
|
||||
}
|
||||
|
||||
// Check for media gallery
|
||||
if (this.matchesPattern(className, this.patterns.media) ||
|
||||
element.querySelectorAll('img, video').length > 3) {
|
||||
scores.media += 5;
|
||||
}
|
||||
}
|
||||
|
||||
// Determine primary content type
|
||||
const maxScore = Math.max(...Object.values(scores));
|
||||
if (maxScore === 0) return 'unknown';
|
||||
|
||||
for (const [type, score] of Object.entries(scores)) {
|
||||
if (score === maxScore) {
|
||||
return type;
|
||||
}
|
||||
}
|
||||
|
||||
return 'mixed';
|
||||
}
|
||||
|
||||
buildHierarchy(elements) {
|
||||
const hierarchy = {
|
||||
root: null,
|
||||
levels: [],
|
||||
headingStructure: []
|
||||
};
|
||||
|
||||
// Find common ancestor
|
||||
if (elements.length > 0) {
|
||||
hierarchy.root = this.findCommonAncestor(elements);
|
||||
}
|
||||
|
||||
// Build heading hierarchy
|
||||
const headings = [];
|
||||
for (const element of elements) {
|
||||
const foundHeadings = element.querySelectorAll('h1, h2, h3, h4, h5, h6');
|
||||
headings.push(...Array.from(foundHeadings));
|
||||
}
|
||||
|
||||
// Sort headings by document position
|
||||
headings.sort((a, b) => {
|
||||
const position = a.compareDocumentPosition(b);
|
||||
if (position & Node.DOCUMENT_POSITION_FOLLOWING) {
|
||||
return -1;
|
||||
} else if (position & Node.DOCUMENT_POSITION_PRECEDING) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
|
||||
// Build heading structure
|
||||
let currentLevel = 0;
|
||||
const stack = [];
|
||||
|
||||
for (const heading of headings) {
|
||||
const level = parseInt(heading.tagName.substring(1));
|
||||
const item = {
|
||||
level,
|
||||
text: heading.textContent.trim(),
|
||||
element: heading,
|
||||
children: []
|
||||
};
|
||||
|
||||
// Find parent in stack
|
||||
while (stack.length > 0 && stack[stack.length - 1].level >= level) {
|
||||
stack.pop();
|
||||
}
|
||||
|
||||
if (stack.length > 0) {
|
||||
stack[stack.length - 1].children.push(item);
|
||||
} else {
|
||||
hierarchy.headingStructure.push(item);
|
||||
}
|
||||
|
||||
stack.push(item);
|
||||
}
|
||||
|
||||
return hierarchy;
|
||||
}
|
||||
|
||||
collectMediaAssets(elements) {
|
||||
const media = {
|
||||
images: [],
|
||||
videos: [],
|
||||
iframes: [],
|
||||
audio: []
|
||||
};
|
||||
|
||||
for (const element of elements) {
|
||||
// Collect images
|
||||
const images = element.querySelectorAll('img');
|
||||
for (const img of images) {
|
||||
media.images.push({
|
||||
src: img.src,
|
||||
alt: img.alt,
|
||||
title: img.title,
|
||||
width: img.width,
|
||||
height: img.height,
|
||||
element: img
|
||||
});
|
||||
}
|
||||
|
||||
// Collect videos
|
||||
const videos = element.querySelectorAll('video');
|
||||
for (const video of videos) {
|
||||
media.videos.push({
|
||||
src: video.src,
|
||||
poster: video.poster,
|
||||
width: video.width,
|
||||
height: video.height,
|
||||
element: video
|
||||
});
|
||||
}
|
||||
|
||||
// Collect iframes
|
||||
const iframes = element.querySelectorAll('iframe');
|
||||
for (const iframe of iframes) {
|
||||
media.iframes.push({
|
||||
src: iframe.src,
|
||||
width: iframe.width,
|
||||
height: iframe.height,
|
||||
title: iframe.title,
|
||||
element: iframe
|
||||
});
|
||||
}
|
||||
|
||||
// Collect audio
|
||||
const audios = element.querySelectorAll('audio');
|
||||
for (const audio of audios) {
|
||||
media.audio.push({
|
||||
src: audio.src,
|
||||
element: audio
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return media;
|
||||
}
|
||||
|
||||
calculateTextDensity(elements) {
|
||||
let totalText = 0;
|
||||
let totalElements = 0;
|
||||
let linkText = 0;
|
||||
let codeText = 0;
|
||||
|
||||
for (const element of elements) {
|
||||
const stats = this.getTextStats(element);
|
||||
totalText += stats.textLength;
|
||||
totalElements += stats.elementCount;
|
||||
linkText += stats.linkTextLength;
|
||||
codeText += stats.codeTextLength;
|
||||
}
|
||||
|
||||
return {
|
||||
textLength: totalText,
|
||||
elementCount: totalElements,
|
||||
averageTextPerElement: totalElements > 0 ? totalText / totalElements : 0,
|
||||
linkDensity: totalText > 0 ? linkText / totalText : 0,
|
||||
codeDensity: totalText > 0 ? codeText / totalText : 0
|
||||
};
|
||||
}
|
||||
|
||||
getTextStats(element, visited = new Set()) {
|
||||
if (visited.has(element)) {
|
||||
return { textLength: 0, elementCount: 0, linkTextLength: 0, codeTextLength: 0 };
|
||||
}
|
||||
visited.add(element);
|
||||
|
||||
let stats = {
|
||||
textLength: 0,
|
||||
elementCount: 1,
|
||||
linkTextLength: 0,
|
||||
codeTextLength: 0
|
||||
};
|
||||
|
||||
// Get direct text content
|
||||
for (const node of element.childNodes) {
|
||||
if (node.nodeType === Node.TEXT_NODE) {
|
||||
const text = node.textContent.trim();
|
||||
stats.textLength += text.length;
|
||||
|
||||
// Check if this text is within a link
|
||||
if (element.tagName === 'A') {
|
||||
stats.linkTextLength += text.length;
|
||||
}
|
||||
|
||||
// Check if this text is within code
|
||||
if (['CODE', 'PRE'].includes(element.tagName)) {
|
||||
stats.codeTextLength += text.length;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process children
|
||||
for (const child of element.children) {
|
||||
const childStats = this.getTextStats(child, visited);
|
||||
stats.textLength += childStats.textLength;
|
||||
stats.elementCount += childStats.elementCount;
|
||||
stats.linkTextLength += childStats.linkTextLength;
|
||||
stats.codeTextLength += childStats.codeTextLength;
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
identifySemanticRegions(elements) {
|
||||
const regions = {
|
||||
headers: [],
|
||||
navigation: [],
|
||||
main: [],
|
||||
sidebars: [],
|
||||
footers: [],
|
||||
articles: []
|
||||
};
|
||||
|
||||
for (const element of elements) {
|
||||
// Check element and its ancestors for semantic regions
|
||||
let current = element;
|
||||
while (current) {
|
||||
const tagName = current.tagName;
|
||||
const className = current.className.toLowerCase();
|
||||
const role = current.getAttribute('role');
|
||||
|
||||
// Check semantic HTML5 elements
|
||||
if (tagName === 'HEADER' || role === 'banner') {
|
||||
regions.headers.push(current);
|
||||
} else if (tagName === 'NAV' || role === 'navigation') {
|
||||
regions.navigation.push(current);
|
||||
} else if (tagName === 'MAIN' || role === 'main') {
|
||||
regions.main.push(current);
|
||||
} else if (tagName === 'ASIDE' || role === 'complementary') {
|
||||
regions.sidebars.push(current);
|
||||
} else if (tagName === 'FOOTER' || role === 'contentinfo') {
|
||||
regions.footers.push(current);
|
||||
} else if (tagName === 'ARTICLE' || role === 'article') {
|
||||
regions.articles.push(current);
|
||||
}
|
||||
|
||||
// Check class patterns
|
||||
if (this.matchesPattern(className, this.patterns.header)) {
|
||||
regions.headers.push(current);
|
||||
} else if (this.matchesPattern(className, this.patterns.navigation)) {
|
||||
regions.navigation.push(current);
|
||||
} else if (this.matchesPattern(className, this.patterns.sidebar)) {
|
||||
regions.sidebars.push(current);
|
||||
} else if (this.matchesPattern(className, this.patterns.footer)) {
|
||||
regions.footers.push(current);
|
||||
}
|
||||
|
||||
current = current.parentElement;
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate
|
||||
for (const key of Object.keys(regions)) {
|
||||
regions[key] = Array.from(new Set(regions[key]));
|
||||
}
|
||||
|
||||
return regions;
|
||||
}
|
||||
|
||||
analyzeRelationships(elements) {
|
||||
const relationships = {
|
||||
siblings: [],
|
||||
parents: [],
|
||||
children: [],
|
||||
relatedByClass: new Map(),
|
||||
relatedByStructure: []
|
||||
};
|
||||
|
||||
// Find sibling relationships
|
||||
for (let i = 0; i < elements.length; i++) {
|
||||
for (let j = i + 1; j < elements.length; j++) {
|
||||
if (elements[i].parentElement === elements[j].parentElement) {
|
||||
relationships.siblings.push([elements[i], elements[j]]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Find parent-child relationships
|
||||
for (const element of elements) {
|
||||
for (const other of elements) {
|
||||
if (element !== other) {
|
||||
if (element.contains(other)) {
|
||||
relationships.parents.push({ parent: element, child: other });
|
||||
} else if (other.contains(element)) {
|
||||
relationships.children.push({ parent: other, child: element });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Group by similar classes
|
||||
for (const element of elements) {
|
||||
const classes = Array.from(element.classList);
|
||||
for (const className of classes) {
|
||||
if (!relationships.relatedByClass.has(className)) {
|
||||
relationships.relatedByClass.set(className, []);
|
||||
}
|
||||
relationships.relatedByClass.get(className).push(element);
|
||||
}
|
||||
}
|
||||
|
||||
// Find structurally similar elements
|
||||
for (let i = 0; i < elements.length; i++) {
|
||||
for (let j = i + 1; j < elements.length; j++) {
|
||||
if (this.areStructurallySimilar(elements[i], elements[j])) {
|
||||
relationships.relatedByStructure.push([elements[i], elements[j]]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return relationships;
|
||||
}
|
||||
|
||||
areStructurallySimilar(element1, element2) {
|
||||
// Same tag name
|
||||
if (element1.tagName !== element2.tagName) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Similar class structure
|
||||
const classes1 = Array.from(element1.classList).sort();
|
||||
const classes2 = Array.from(element2.classList).sort();
|
||||
|
||||
// At least 50% overlap in classes
|
||||
const intersection = classes1.filter(c => classes2.includes(c));
|
||||
const union = Array.from(new Set([...classes1, ...classes2]));
|
||||
|
||||
if (union.length > 0 && intersection.length / union.length >= 0.5) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Similar child structure
|
||||
if (element1.children.length === element2.children.length) {
|
||||
const childTags1 = Array.from(element1.children).map(c => c.tagName).sort();
|
||||
const childTags2 = Array.from(element2.children).map(c => c.tagName).sort();
|
||||
|
||||
if (JSON.stringify(childTags1) === JSON.stringify(childTags2)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
extractMetadata(elements) {
|
||||
const metadata = {
|
||||
title: null,
|
||||
description: null,
|
||||
author: null,
|
||||
date: null,
|
||||
tags: [],
|
||||
microdata: []
|
||||
};
|
||||
|
||||
for (const element of elements) {
|
||||
// Look for title
|
||||
const h1 = element.querySelector('h1');
|
||||
if (h1 && !metadata.title) {
|
||||
metadata.title = h1.textContent.trim();
|
||||
}
|
||||
|
||||
// Look for meta information
|
||||
const metaElements = element.querySelectorAll('[itemprop], [property], [name]');
|
||||
for (const meta of metaElements) {
|
||||
const prop = meta.getAttribute('itemprop') ||
|
||||
meta.getAttribute('property') ||
|
||||
meta.getAttribute('name');
|
||||
const content = meta.getAttribute('content') || meta.textContent.trim();
|
||||
|
||||
if (prop && content) {
|
||||
if (prop.includes('author')) {
|
||||
metadata.author = content;
|
||||
} else if (prop.includes('date') || prop.includes('time')) {
|
||||
metadata.date = content;
|
||||
} else if (prop.includes('description')) {
|
||||
metadata.description = content;
|
||||
} else if (prop.includes('tag') || prop.includes('keyword')) {
|
||||
metadata.tags.push(content);
|
||||
}
|
||||
|
||||
metadata.microdata.push({ property: prop, value: content });
|
||||
}
|
||||
}
|
||||
|
||||
// Look for time elements
|
||||
const timeElements = element.querySelectorAll('time');
|
||||
for (const time of timeElements) {
|
||||
if (!metadata.date && time.dateTime) {
|
||||
metadata.date = time.dateTime;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
determineLayout(elements) {
|
||||
// Check if elements form a grid
|
||||
const positions = elements.map(el => {
|
||||
const rect = el.getBoundingClientRect();
|
||||
return { x: rect.left, y: rect.top, width: rect.width, height: rect.height };
|
||||
});
|
||||
|
||||
// Check for grid layout (multiple elements on same row)
|
||||
const rows = new Map();
|
||||
for (const pos of positions) {
|
||||
const row = Math.round(pos.y / 10) * 10; // Round to nearest 10px
|
||||
if (!rows.has(row)) {
|
||||
rows.set(row, []);
|
||||
}
|
||||
rows.get(row).push(pos);
|
||||
}
|
||||
|
||||
// If multiple elements share rows, it's likely a grid
|
||||
const hasGrid = Array.from(rows.values()).some(row => row.length > 1);
|
||||
|
||||
if (hasGrid) {
|
||||
return 'grid';
|
||||
}
|
||||
|
||||
// Check for mixed layout (significant variation in widths)
|
||||
const widths = positions.map(p => p.width);
|
||||
const avgWidth = widths.reduce((a, b) => a + b, 0) / widths.length;
|
||||
const variance = widths.reduce((sum, w) => sum + Math.pow(w - avgWidth, 2), 0) / widths.length;
|
||||
const stdDev = Math.sqrt(variance);
|
||||
|
||||
if (stdDev / avgWidth > 0.3) {
|
||||
return 'mixed';
|
||||
}
|
||||
|
||||
return 'linear';
|
||||
}
|
||||
|
||||
calculateMaxDepth(elements) {
|
||||
let maxDepth = 0;
|
||||
|
||||
for (const element of elements) {
|
||||
const depth = this.getElementDepth(element);
|
||||
maxDepth = Math.max(maxDepth, depth);
|
||||
}
|
||||
|
||||
return maxDepth;
|
||||
}
|
||||
|
||||
getElementDepth(element, depth = 0) {
|
||||
if (element.children.length === 0) {
|
||||
return depth;
|
||||
}
|
||||
|
||||
let maxChildDepth = depth;
|
||||
for (const child of element.children) {
|
||||
const childDepth = this.getElementDepth(child, depth + 1);
|
||||
maxChildDepth = Math.max(maxChildDepth, childDepth);
|
||||
}
|
||||
|
||||
return maxChildDepth;
|
||||
}
|
||||
|
||||
findCommonAncestor(elements) {
|
||||
if (elements.length === 0) return null;
|
||||
if (elements.length === 1) return elements[0].parentElement;
|
||||
|
||||
// Start with the first element's ancestors
|
||||
let ancestor = elements[0];
|
||||
const ancestors = [];
|
||||
|
||||
while (ancestor) {
|
||||
ancestors.push(ancestor);
|
||||
ancestor = ancestor.parentElement;
|
||||
}
|
||||
|
||||
// Find the deepest common ancestor
|
||||
for (const ancestorCandidate of ancestors) {
|
||||
let isCommon = true;
|
||||
|
||||
for (const element of elements) {
|
||||
if (!ancestorCandidate.contains(element)) {
|
||||
isCommon = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (isCommon) {
|
||||
return ancestorCandidate;
|
||||
}
|
||||
}
|
||||
|
||||
return document.body;
|
||||
}
|
||||
|
||||
matchesPattern(text, patterns) {
|
||||
return patterns.some(pattern => text.includes(pattern));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
class MarkdownConverter {
|
||||
constructor() {
|
||||
// Conversion handlers for different element types
|
||||
this.converters = {
|
||||
'H1': async (el, ctx) => await this.convertHeading(el, 1, ctx),
|
||||
'H2': async (el, ctx) => await this.convertHeading(el, 2, ctx),
|
||||
'H3': async (el, ctx) => await this.convertHeading(el, 3, ctx),
|
||||
'H4': async (el, ctx) => await this.convertHeading(el, 4, ctx),
|
||||
'H5': async (el, ctx) => await this.convertHeading(el, 5, ctx),
|
||||
'H6': async (el, ctx) => await this.convertHeading(el, 6, ctx),
|
||||
'P': async (el, ctx) => await this.convertParagraph(el, ctx),
|
||||
'A': async (el, ctx) => await this.convertLink(el, ctx),
|
||||
'IMG': async (el, ctx) => await this.convertImage(el, ctx),
|
||||
'UL': async (el, ctx) => await this.convertList(el, 'ul', ctx),
|
||||
'OL': async (el, ctx) => await this.convertList(el, 'ol', ctx),
|
||||
'LI': async (el, ctx) => await this.convertListItem(el, ctx),
|
||||
'TABLE': async (el, ctx) => await this.convertTable(el, ctx),
|
||||
'BLOCKQUOTE': async (el, ctx) => await this.convertBlockquote(el, ctx),
|
||||
'PRE': async (el, ctx) => await this.convertPreformatted(el, ctx),
|
||||
'CODE': async (el, ctx) => await this.convertCode(el, ctx),
|
||||
'HR': async (el, ctx) => '\n---\n',
|
||||
'BR': async (el, ctx) => ' \n',
|
||||
'STRONG': async (el, ctx) => `**${await this.getTextContent(el, ctx)}**`,
|
||||
'B': async (el, ctx) => `**${await this.getTextContent(el, ctx)}**`,
|
||||
'EM': async (el, ctx) => `*${await this.getTextContent(el, ctx)}*`,
|
||||
'I': async (el, ctx) => `*${await this.getTextContent(el, ctx)}*`,
|
||||
'DEL': async (el, ctx) => `~~${await this.getTextContent(el, ctx)}~~`,
|
||||
'S': async (el, ctx) => `~~${await this.getTextContent(el, ctx)}~~`,
|
||||
'DIV': async (el, ctx) => await this.convertDiv(el, ctx),
|
||||
'SPAN': async (el, ctx) => await this.convertSpan(el, ctx),
|
||||
'ARTICLE': async (el, ctx) => await this.convertArticle(el, ctx),
|
||||
'SECTION': async (el, ctx) => await this.convertSection(el, ctx),
|
||||
'FIGURE': async (el, ctx) => await this.convertFigure(el, ctx),
|
||||
'FIGCAPTION': async (el, ctx) => await this.convertFigCaption(el, ctx),
|
||||
'VIDEO': async (el, ctx) => await this.convertVideo(el, ctx),
|
||||
'IFRAME': async (el, ctx) => await this.convertIframe(el, ctx),
|
||||
'DL': async (el, ctx) => await this.convertDefinitionList(el, ctx),
|
||||
'DT': async (el, ctx) => await this.convertDefinitionTerm(el, ctx),
|
||||
'DD': async (el, ctx) => await this.convertDefinitionDescription(el, ctx),
|
||||
'TR': async (el, ctx) => await this.convertTableRow(el, ctx)
|
||||
};
|
||||
|
||||
// Maintain context during conversion
|
||||
this.conversionContext = {
|
||||
listDepth: 0,
|
||||
inTable: false,
|
||||
inCode: false,
|
||||
preserveWhitespace: false,
|
||||
references: [],
|
||||
imageCount: 0,
|
||||
linkCount: 0
|
||||
};
|
||||
}
|
||||
|
||||
async convert(elements, options = {}) {
|
||||
// Reset context
|
||||
this.resetContext();
|
||||
|
||||
// Apply options
|
||||
this.options = {
|
||||
includeImages: true,
|
||||
preserveTables: true,
|
||||
keepCodeFormatting: true,
|
||||
simplifyLayout: false,
|
||||
preserveLinks: true,
|
||||
...options
|
||||
};
|
||||
|
||||
// Convert elements
|
||||
const markdownParts = [];
|
||||
|
||||
for (const element of elements) {
|
||||
const markdown = await this.convertElement(element, this.conversionContext);
|
||||
if (markdown.trim()) {
|
||||
markdownParts.push(markdown);
|
||||
}
|
||||
}
|
||||
|
||||
// Join parts with appropriate spacing
|
||||
let result = markdownParts.join('\n\n');
|
||||
|
||||
// Add references if using reference-style links
|
||||
if (this.conversionContext.references.length > 0) {
|
||||
result += '\n\n' + this.generateReferences();
|
||||
}
|
||||
|
||||
// Post-process to clean up
|
||||
result = this.postProcess(result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
resetContext() {
|
||||
this.conversionContext = {
|
||||
listDepth: 0,
|
||||
inTable: false,
|
||||
inCode: false,
|
||||
preserveWhitespace: false,
|
||||
references: [],
|
||||
imageCount: 0,
|
||||
linkCount: 0
|
||||
};
|
||||
}
|
||||
|
||||
async convertElement(element, context) {
|
||||
// Skip hidden elements
|
||||
if (this.isHidden(element)) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Skip script and style elements
|
||||
if (['SCRIPT', 'STYLE', 'NOSCRIPT'].includes(element.tagName)) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Get converter for this element type
|
||||
const converter = this.converters[element.tagName];
|
||||
|
||||
if (converter) {
|
||||
return await converter(element, context);
|
||||
} else {
|
||||
// For unknown elements, process children
|
||||
return await this.processChildren(element, context);
|
||||
}
|
||||
}
|
||||
|
||||
async processChildren(element, context) {
|
||||
const parts = [];
|
||||
|
||||
for (const child of element.childNodes) {
|
||||
if (child.nodeType === Node.TEXT_NODE) {
|
||||
const text = this.processTextNode(child, context);
|
||||
if (text) {
|
||||
parts.push(text);
|
||||
}
|
||||
} else if (child.nodeType === Node.ELEMENT_NODE) {
|
||||
const markdown = await this.convertElement(child, context);
|
||||
if (markdown) {
|
||||
parts.push(markdown);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('');
|
||||
}
|
||||
|
||||
processTextNode(node, context) {
|
||||
let text = node.textContent;
|
||||
|
||||
// Preserve whitespace in code blocks
|
||||
if (!context.preserveWhitespace && !context.inCode) {
|
||||
// Normalize whitespace
|
||||
text = text.replace(/\s+/g, ' ');
|
||||
|
||||
// Trim if at block boundaries
|
||||
if (this.isBlockBoundary(node.previousSibling)) {
|
||||
text = text.trimStart();
|
||||
}
|
||||
if (this.isBlockBoundary(node.nextSibling)) {
|
||||
text = text.trimEnd();
|
||||
}
|
||||
}
|
||||
|
||||
// Escape markdown characters
|
||||
if (!context.inCode) {
|
||||
text = this.escapeMarkdown(text);
|
||||
}
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
isBlockBoundary(node) {
|
||||
if (!node || node.nodeType !== Node.ELEMENT_NODE) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const blockElements = [
|
||||
'DIV', 'P', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6',
|
||||
'UL', 'OL', 'LI', 'BLOCKQUOTE', 'PRE', 'TABLE',
|
||||
'HR', 'ARTICLE', 'SECTION', 'HEADER', 'FOOTER',
|
||||
'NAV', 'ASIDE', 'MAIN'
|
||||
];
|
||||
|
||||
return blockElements.includes(node.tagName);
|
||||
}
|
||||
|
||||
escapeMarkdown(text) {
|
||||
// In text-only mode, don't escape characters
|
||||
if (this.options.textOnly) {
|
||||
return text;
|
||||
}
|
||||
|
||||
// Escape special markdown characters
|
||||
return text
|
||||
.replace(/\\/g, '\\\\')
|
||||
.replace(/\*/g, '\\*')
|
||||
.replace(/_/g, '\\_')
|
||||
.replace(/\[/g, '\\[')
|
||||
.replace(/\]/g, '\\]')
|
||||
.replace(/\(/g, '\\(')
|
||||
.replace(/\)/g, '\\)')
|
||||
.replace(/\#/g, '\\#')
|
||||
.replace(/\+/g, '\\+')
|
||||
.replace(/\-/g, '\\-')
|
||||
.replace(/\./g, '\\.')
|
||||
.replace(/\!/g, '\\!')
|
||||
.replace(/\|/g, '\\|');
|
||||
}
|
||||
|
||||
async convertHeading(element, level, context) {
|
||||
const text = await this.getTextContent(element, context);
|
||||
return '#'.repeat(level) + ' ' + text + '\n';
|
||||
}
|
||||
|
||||
async convertParagraph(element, context) {
|
||||
const content = await this.processChildren(element, context);
|
||||
return content.trim() ? content + '\n' : '';
|
||||
}
|
||||
|
||||
async convertLink(element, context) {
|
||||
if (!this.options.preserveLinks || this.options.textOnly) {
|
||||
return await this.getTextContent(element, context);
|
||||
}
|
||||
|
||||
const text = await this.getTextContent(element, context);
|
||||
const href = element.getAttribute('href');
|
||||
const title = element.getAttribute('title');
|
||||
|
||||
if (!href) {
|
||||
return text;
|
||||
}
|
||||
|
||||
// Convert relative URLs to absolute
|
||||
const absoluteUrl = this.makeAbsoluteUrl(href);
|
||||
|
||||
// Use reference-style links for cleaner markdown
|
||||
if (text && absoluteUrl) {
|
||||
if (title) {
|
||||
return `[${text}](${absoluteUrl} "${title}")`;
|
||||
} else {
|
||||
return `[${text}](${absoluteUrl})`;
|
||||
}
|
||||
}
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
async convertImage(element, context) {
|
||||
if (!this.options.includeImages || this.options.textOnly) {
|
||||
// In text-only mode, return alt text if available
|
||||
if (this.options.textOnly) {
|
||||
const alt = element.getAttribute('alt');
|
||||
return alt ? `[Image: ${alt}]` : '';
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
const src = element.getAttribute('src');
|
||||
const alt = element.getAttribute('alt') || '';
|
||||
const title = element.getAttribute('title');
|
||||
|
||||
if (!src) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Convert relative URLs to absolute
|
||||
const absoluteUrl = this.makeAbsoluteUrl(src);
|
||||
|
||||
if (title) {
|
||||
return ``;
|
||||
} else {
|
||||
return ``;
|
||||
}
|
||||
}
|
||||
|
||||
async convertList(element, type, context) {
|
||||
const oldDepth = context.listDepth;
|
||||
context.listDepth++;
|
||||
|
||||
const items = [];
|
||||
for (const child of element.children) {
|
||||
if (child.tagName === 'LI') {
|
||||
const markdown = await this.convertListItem(child, { ...context, listType: type });
|
||||
if (markdown) {
|
||||
items.push(markdown);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
context.listDepth = oldDepth;
|
||||
|
||||
return items.join('\n') + (context.listDepth === 0 ? '\n' : '');
|
||||
}
|
||||
|
||||
async convertListItem(element, context) {
|
||||
const indent = ' '.repeat(Math.max(0, context.listDepth - 1));
|
||||
const bullet = context.listType === 'ol' ? '1.' : '-';
|
||||
const content = (await this.processChildren(element, context)).trim();
|
||||
|
||||
return `${indent}${bullet} ${content}`;
|
||||
}
|
||||
|
||||
async convertTable(element, context) {
|
||||
if (!this.options.preserveTables || this.options.textOnly) {
|
||||
// Fallback to simple text representation
|
||||
return await this.convertTableToText(element, context);
|
||||
}
|
||||
|
||||
const rows = [];
|
||||
const headerRows = [];
|
||||
let maxCols = 0;
|
||||
|
||||
// Process table rows
|
||||
for (const child of element.children) {
|
||||
if (child.tagName === 'THEAD') {
|
||||
for (const row of child.children) {
|
||||
if (row.tagName === 'TR') {
|
||||
const cells = await this.processTableRow(row, context);
|
||||
headerRows.push(cells);
|
||||
maxCols = Math.max(maxCols, cells.length);
|
||||
}
|
||||
}
|
||||
} else if (child.tagName === 'TBODY') {
|
||||
for (const row of child.children) {
|
||||
if (row.tagName === 'TR') {
|
||||
const cells = await this.processTableRow(row, context);
|
||||
rows.push(cells);
|
||||
maxCols = Math.max(maxCols, cells.length);
|
||||
}
|
||||
}
|
||||
} else if (child.tagName === 'TR') {
|
||||
const cells = await this.processTableRow(child, context);
|
||||
rows.push(cells);
|
||||
maxCols = Math.max(maxCols, cells.length);
|
||||
}
|
||||
}
|
||||
|
||||
// Build markdown table
|
||||
const markdownRows = [];
|
||||
|
||||
// Add headers
|
||||
if (headerRows.length > 0) {
|
||||
for (const headerRow of headerRows) {
|
||||
const paddedRow = this.padTableRow(headerRow, maxCols);
|
||||
markdownRows.push('| ' + paddedRow.join(' | ') + ' |');
|
||||
}
|
||||
|
||||
// Add separator
|
||||
const separator = Array(maxCols).fill('---');
|
||||
markdownRows.push('| ' + separator.join(' | ') + ' |');
|
||||
}
|
||||
|
||||
// Add body rows
|
||||
for (const row of rows) {
|
||||
const paddedRow = this.padTableRow(row, maxCols);
|
||||
markdownRows.push('| ' + paddedRow.join(' | ') + ' |');
|
||||
}
|
||||
|
||||
return markdownRows.join('\n') + '\n';
|
||||
}
|
||||
|
||||
async processTableRow(row, context) {
|
||||
const cells = [];
|
||||
|
||||
for (const cell of row.children) {
|
||||
if (cell.tagName === 'TD' || cell.tagName === 'TH') {
|
||||
const content = (await this.getTextContent(cell, context)).trim();
|
||||
cells.push(content);
|
||||
}
|
||||
}
|
||||
|
||||
return cells;
|
||||
}
|
||||
|
||||
async convertTableRow(element, context) {
|
||||
// Convert a single table row to markdown
|
||||
if (this.options.textOnly) {
|
||||
const cells = await this.processTableRow(element, context);
|
||||
return cells.join(' ');
|
||||
}
|
||||
|
||||
// For non-text-only mode, create a simple table representation
|
||||
const cells = await this.processTableRow(element, context);
|
||||
return '| ' + cells.join(' | ') + ' |';
|
||||
}
|
||||
|
||||
padTableRow(row, targetLength) {
|
||||
const padded = [...row];
|
||||
while (padded.length < targetLength) {
|
||||
padded.push('');
|
||||
}
|
||||
return padded;
|
||||
}
|
||||
|
||||
async convertTableToText(element, context) {
|
||||
// Convert table to clean text representation
|
||||
const lines = [];
|
||||
const rows = element.querySelectorAll('tr');
|
||||
|
||||
for (const row of rows) {
|
||||
const cells = row.querySelectorAll('td, th');
|
||||
const cellTexts = [];
|
||||
|
||||
for (const cell of cells) {
|
||||
const text = (await this.getTextContent(cell, context)).trim();
|
||||
if (text) {
|
||||
cellTexts.push(text);
|
||||
}
|
||||
}
|
||||
|
||||
if (cellTexts.length > 0) {
|
||||
// Join cells with space, handling common patterns
|
||||
lines.push(cellTexts.join(' '));
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async convertBlockquote(element, context) {
|
||||
const lines = (await this.processChildren(element, context)).trim().split('\n');
|
||||
return lines.map(line => '> ' + line).join('\n') + '\n';
|
||||
}
|
||||
|
||||
async convertPreformatted(element, context) {
|
||||
const oldInCode = context.inCode;
|
||||
const oldPreserveWhitespace = context.preserveWhitespace;
|
||||
|
||||
context.inCode = true;
|
||||
context.preserveWhitespace = true;
|
||||
|
||||
let content = '';
|
||||
let language = '';
|
||||
|
||||
// Check if this is a code block with language
|
||||
const codeElement = element.querySelector('code');
|
||||
if (codeElement) {
|
||||
// Try to detect language from class
|
||||
const className = codeElement.className;
|
||||
const langMatch = className.match(/language-(\w+)/);
|
||||
if (langMatch) {
|
||||
language = langMatch[1];
|
||||
}
|
||||
|
||||
content = codeElement.textContent;
|
||||
} else {
|
||||
content = element.textContent;
|
||||
}
|
||||
|
||||
context.inCode = oldInCode;
|
||||
context.preserveWhitespace = oldPreserveWhitespace;
|
||||
|
||||
// Use fenced code blocks
|
||||
return '```' + language + '\n' + content + '\n```\n';
|
||||
}
|
||||
|
||||
async convertCode(element, context) {
|
||||
if (element.parentElement && element.parentElement.tagName === 'PRE') {
|
||||
// Already handled by convertPreformatted
|
||||
return element.textContent;
|
||||
}
|
||||
|
||||
const content = element.textContent;
|
||||
return '`' + content + '`';
|
||||
}
|
||||
|
||||
async convertDiv(element, context) {
|
||||
// Check for special div types
|
||||
if (element.className.includes('code-block') ||
|
||||
element.className.includes('highlight')) {
|
||||
return await this.convertPreformatted(element, context);
|
||||
}
|
||||
|
||||
const content = await this.processChildren(element, context);
|
||||
return content.trim() ? content + '\n' : '';
|
||||
}
|
||||
|
||||
async convertSpan(element, context) {
|
||||
// Check for special span types
|
||||
if (element.className.includes('code') ||
|
||||
element.className.includes('inline-code')) {
|
||||
return this.convertCode(element, context);
|
||||
}
|
||||
|
||||
return await this.processChildren(element, context);
|
||||
}
|
||||
|
||||
async convertArticle(element, context) {
|
||||
const content = await this.processChildren(element, context);
|
||||
return content.trim() ? content + '\n' : '';
|
||||
}
|
||||
|
||||
async convertSection(element, context) {
|
||||
const content = await this.processChildren(element, context);
|
||||
return content.trim() ? content + '\n' : '';
|
||||
}
|
||||
|
||||
async convertFigure(element, context) {
|
||||
const content = await this.processChildren(element, context);
|
||||
return content.trim() ? content + '\n' : '';
|
||||
}
|
||||
|
||||
async convertFigCaption(element, context) {
|
||||
const caption = await this.getTextContent(element, context);
|
||||
return caption ? '\n*' + caption + '*\n' : '';
|
||||
}
|
||||
|
||||
async convertVideo(element, context) {
|
||||
const title = element.getAttribute('title') || 'Video';
|
||||
|
||||
if (this.options.textOnly) {
|
||||
return `[Video: ${title}]`;
|
||||
}
|
||||
|
||||
const src = element.getAttribute('src');
|
||||
const poster = element.getAttribute('poster');
|
||||
|
||||
if (!src) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Convert to markdown with poster image if available
|
||||
if (poster) {
|
||||
const absolutePoster = this.makeAbsoluteUrl(poster);
|
||||
const absoluteSrc = this.makeAbsoluteUrl(src);
|
||||
return `[](${absoluteSrc})`;
|
||||
} else {
|
||||
const absoluteSrc = this.makeAbsoluteUrl(src);
|
||||
return `[${title}](${absoluteSrc})`;
|
||||
}
|
||||
}
|
||||
|
||||
async convertIframe(element, context) {
|
||||
const title = element.getAttribute('title') || 'Embedded content';
|
||||
|
||||
if (this.options.textOnly) {
|
||||
const src = element.getAttribute('src') || '';
|
||||
if (src.includes('youtube.com') || src.includes('youtu.be')) {
|
||||
return `[Video: ${title}]`;
|
||||
} else if (src.includes('vimeo.com')) {
|
||||
return `[Video: ${title}]`;
|
||||
} else {
|
||||
return `[Embedded: ${title}]`;
|
||||
}
|
||||
}
|
||||
|
||||
const src = element.getAttribute('src');
|
||||
if (!src) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Check for common embeds
|
||||
if (src.includes('youtube.com') || src.includes('youtu.be')) {
|
||||
return `[▶️ ${title}](${src})`;
|
||||
} else if (src.includes('vimeo.com')) {
|
||||
return `[▶️ ${title}](${src})`;
|
||||
} else {
|
||||
return `[${title}](${src})`;
|
||||
}
|
||||
}
|
||||
|
||||
async convertDefinitionList(element, context) {
|
||||
return await this.processChildren(element, context) + '\n';
|
||||
}
|
||||
|
||||
async convertDefinitionTerm(element, context) {
|
||||
const term = await this.getTextContent(element, context);
|
||||
return '**' + term + '**\n';
|
||||
}
|
||||
|
||||
async convertDefinitionDescription(element, context) {
|
||||
const description = await this.processChildren(element, context);
|
||||
return ': ' + description + '\n';
|
||||
}
|
||||
|
||||
async getTextContent(element, context) {
|
||||
// Special handling for elements that might contain other markdown
|
||||
if (context.inCode) {
|
||||
return element.textContent;
|
||||
}
|
||||
|
||||
return await this.processChildren(element, context);
|
||||
}
|
||||
|
||||
makeAbsoluteUrl(url) {
|
||||
if (!url) return '';
|
||||
|
||||
try {
|
||||
// Check if already absolute
|
||||
if (url.startsWith('http://') || url.startsWith('https://')) {
|
||||
return url;
|
||||
}
|
||||
|
||||
// Handle protocol-relative URLs
|
||||
if (url.startsWith('//')) {
|
||||
return window.location.protocol + url;
|
||||
}
|
||||
|
||||
// Convert relative to absolute
|
||||
const base = window.location.origin;
|
||||
const path = window.location.pathname;
|
||||
|
||||
if (url.startsWith('/')) {
|
||||
return base + url;
|
||||
} else {
|
||||
// Relative to current path
|
||||
const pathDir = path.substring(0, path.lastIndexOf('/') + 1);
|
||||
return base + pathDir + url;
|
||||
}
|
||||
} catch (e) {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
isHidden(element) {
|
||||
const style = window.getComputedStyle(element);
|
||||
return style.display === 'none' ||
|
||||
style.visibility === 'hidden' ||
|
||||
style.opacity === '0';
|
||||
}
|
||||
|
||||
generateReferences() {
|
||||
return this.conversionContext.references
|
||||
.map((ref, index) => `[${index + 1}]: ${ref.url}`)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
postProcess(markdown) {
|
||||
// Apply text-only specific processing
|
||||
if (this.options.textOnly) {
|
||||
markdown = this.postProcessTextOnly(markdown);
|
||||
}
|
||||
|
||||
// Clean up excessive newlines
|
||||
markdown = markdown.replace(/\n{3,}/g, '\n\n');
|
||||
|
||||
// Clean up spaces before punctuation
|
||||
markdown = markdown.replace(/ +([.,;:!?])/g, '$1');
|
||||
|
||||
// Ensure proper spacing around headers
|
||||
markdown = markdown.replace(/\n(#{1,6} )/g, '\n\n$1');
|
||||
markdown = markdown.replace(/(#{1,6} .+)\n(?![\n#])/g, '$1\n\n');
|
||||
|
||||
// Clean up list spacing
|
||||
markdown = markdown.replace(/\n\n(-|\d+\.) /g, '\n$1 ');
|
||||
|
||||
// Trim final result
|
||||
return markdown.trim();
|
||||
}
|
||||
|
||||
postProcessTextOnly(markdown) {
|
||||
// Smart pattern recognition for common formats
|
||||
const lines = markdown.split('\n');
|
||||
const processedLines = [];
|
||||
let inMetadata = false;
|
||||
let currentItem = null;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i].trim();
|
||||
if (!line) {
|
||||
processedLines.push('');
|
||||
continue;
|
||||
}
|
||||
|
||||
// Detect numbered list items (common in HN, Reddit, etc.)
|
||||
const numberPattern = /^(\d+)\.\s*(.+)$/;
|
||||
const numberMatch = line.match(numberPattern);
|
||||
|
||||
if (numberMatch) {
|
||||
// Start of a new numbered item
|
||||
inMetadata = false;
|
||||
currentItem = numberMatch[1];
|
||||
const content = numberMatch[2];
|
||||
|
||||
// Check if content has domain in parentheses
|
||||
const domainPattern = /^(.+?)\s*\(([^)]+)\)\s*(.*)$/;
|
||||
const domainMatch = content.match(domainPattern);
|
||||
|
||||
if (domainMatch) {
|
||||
const [, title, domain, rest] = domainMatch;
|
||||
processedLines.push(`${currentItem}. **${title.trim()}** (${domain})`);
|
||||
if (rest.trim()) {
|
||||
processedLines.push(` ${rest.trim()}`);
|
||||
inMetadata = true;
|
||||
}
|
||||
} else {
|
||||
processedLines.push(`${currentItem}. **${content}**`);
|
||||
}
|
||||
} else if (line.match(/\b(points?|by|ago|hide|comments?)\b/i) && currentItem) {
|
||||
// This looks like metadata for the current item
|
||||
const cleanedLine = line
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/\s*\|\s*/g, ' | ')
|
||||
.trim();
|
||||
processedLines.push(` ${cleanedLine}`);
|
||||
inMetadata = true;
|
||||
} else if (inMetadata && line.length < 100) {
|
||||
// Continue metadata if we're in metadata mode and line is short
|
||||
processedLines.push(` ${line}`);
|
||||
} else {
|
||||
// Regular content
|
||||
inMetadata = false;
|
||||
processedLines.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up the output
|
||||
let result = processedLines.join('\n');
|
||||
|
||||
// Remove excessive blank lines
|
||||
result = result.replace(/\n{3,}/g, '\n\n');
|
||||
|
||||
// Ensure proper spacing after numbered items
|
||||
result = result.replace(/^(\d+\..+)$\n^(?!\s)/gm, '$1\n\n');
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,701 @@
|
||||
class MarkdownExtraction {
|
||||
constructor() {
|
||||
this.selectedElements = new Set();
|
||||
this.highlightBoxes = new Map();
|
||||
this.selectionMode = false;
|
||||
this.toolbar = null;
|
||||
this.markdownPreviewModal = null;
|
||||
this.selectionCounter = 0;
|
||||
this.markdownConverter = null;
|
||||
this.contentAnalyzer = null;
|
||||
|
||||
this.init();
|
||||
}
|
||||
|
||||
async init() {
|
||||
// Initialize dependencies
|
||||
this.markdownConverter = new MarkdownConverter();
|
||||
this.contentAnalyzer = new ContentAnalyzer();
|
||||
|
||||
this.createToolbar();
|
||||
this.setupEventListeners();
|
||||
}
|
||||
|
||||
createToolbar() {
|
||||
// Create floating toolbar
|
||||
this.toolbar = document.createElement('div');
|
||||
this.toolbar.className = 'c4ai-c2c-toolbar';
|
||||
this.toolbar.innerHTML = `
|
||||
<div class="c4ai-toolbar-header">
|
||||
<div class="c4ai-toolbar-dots">
|
||||
<span class="c4ai-dot c4ai-dot-red"></span>
|
||||
<span class="c4ai-dot c4ai-dot-yellow"></span>
|
||||
<span class="c4ai-dot c4ai-dot-green"></span>
|
||||
</div>
|
||||
<span class="c4ai-toolbar-title">Markdown Extraction</span>
|
||||
<button class="c4ai-close-btn" title="Close">×</button>
|
||||
</div>
|
||||
<div class="c4ai-toolbar-content">
|
||||
<div class="c4ai-selection-info">
|
||||
<span class="c4ai-selection-count">0 elements selected</span>
|
||||
<button class="c4ai-clear-btn" title="Clear selection" disabled>Clear</button>
|
||||
</div>
|
||||
<div class="c4ai-toolbar-actions">
|
||||
<button class="c4ai-preview-btn" disabled>Preview Markdown</button>
|
||||
<button class="c4ai-copy-btn" disabled>Copy to Clipboard</button>
|
||||
</div>
|
||||
<div class="c4ai-toolbar-instructions">
|
||||
<p>💡 <strong>Ctrl/Cmd + Click</strong> to select multiple elements</p>
|
||||
<p>📝 Selected elements will be converted to clean markdown</p>
|
||||
<p>⌨️ Press <strong>ESC</strong> to exit</p>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
document.body.appendChild(this.toolbar);
|
||||
makeDraggableByHeader(this.toolbar);
|
||||
|
||||
// Position toolbar
|
||||
this.toolbar.style.position = 'fixed';
|
||||
this.toolbar.style.top = '20px';
|
||||
this.toolbar.style.right = '20px';
|
||||
this.toolbar.style.zIndex = '999999';
|
||||
}
|
||||
|
||||
setupEventListeners() {
|
||||
// Close button
|
||||
this.toolbar.querySelector('.c4ai-close-btn').addEventListener('click', () => {
|
||||
this.deactivate();
|
||||
});
|
||||
|
||||
// Clear selection button
|
||||
this.toolbar.querySelector('.c4ai-clear-btn').addEventListener('click', () => {
|
||||
this.clearSelection();
|
||||
});
|
||||
|
||||
// Preview button
|
||||
this.toolbar.querySelector('.c4ai-preview-btn').addEventListener('click', () => {
|
||||
this.showPreview();
|
||||
});
|
||||
|
||||
// Copy button
|
||||
this.toolbar.querySelector('.c4ai-copy-btn').addEventListener('click', () => {
|
||||
this.copyToClipboard();
|
||||
});
|
||||
|
||||
// Document click handler for element selection
|
||||
this.documentClickHandler = (event) => this.handleElementClick(event);
|
||||
document.addEventListener('click', this.documentClickHandler, true);
|
||||
|
||||
// Prevent default link behavior during selection mode
|
||||
this.linkClickHandler = (event) => {
|
||||
if (event.ctrlKey || event.metaKey) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}
|
||||
};
|
||||
document.addEventListener('click', this.linkClickHandler, true);
|
||||
|
||||
// Hover effect
|
||||
this.documentHoverHandler = (event) => this.handleElementHover(event);
|
||||
document.addEventListener('mouseover', this.documentHoverHandler, true);
|
||||
|
||||
// Remove hover on mouseout
|
||||
this.documentMouseOutHandler = (event) => this.handleElementMouseOut(event);
|
||||
document.addEventListener('mouseout', this.documentMouseOutHandler, true);
|
||||
|
||||
// Keyboard shortcuts
|
||||
this.keyboardHandler = (event) => this.handleKeyboard(event);
|
||||
document.addEventListener('keydown', this.keyboardHandler);
|
||||
}
|
||||
|
||||
handleElementClick(event) {
|
||||
// Check if Ctrl/Cmd is pressed
|
||||
if (!event.ctrlKey && !event.metaKey) return;
|
||||
|
||||
// Prevent default behavior
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
|
||||
const element = event.target;
|
||||
|
||||
// Don't select our own UI elements
|
||||
if (element.closest('.c4ai-c2c-toolbar') ||
|
||||
element.closest('.c4ai-c2c-preview') ||
|
||||
element.closest('.c4ai-highlight-box')) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Toggle element selection
|
||||
if (this.selectedElements.has(element)) {
|
||||
this.deselectElement(element);
|
||||
} else {
|
||||
this.selectElement(element);
|
||||
}
|
||||
|
||||
this.updateUI();
|
||||
}
|
||||
|
||||
handleElementHover(event) {
|
||||
const element = event.target;
|
||||
|
||||
// Don't hover our own UI elements
|
||||
if (element.closest('.c4ai-c2c-toolbar') ||
|
||||
element.closest('.c4ai-c2c-preview') ||
|
||||
element.closest('.c4ai-highlight-box') ||
|
||||
element.hasAttribute('data-c4ai-badge')) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Add hover class
|
||||
element.classList.add('c4ai-hover-candidate');
|
||||
}
|
||||
|
||||
handleElementMouseOut(event) {
|
||||
const element = event.target;
|
||||
element.classList.remove('c4ai-hover-candidate');
|
||||
}
|
||||
|
||||
handleKeyboard(event) {
|
||||
// ESC to deactivate
|
||||
if (event.key === 'Escape') {
|
||||
this.deactivate();
|
||||
}
|
||||
// Ctrl/Cmd + A to select all visible elements
|
||||
else if ((event.ctrlKey || event.metaKey) && event.key === 'a') {
|
||||
event.preventDefault();
|
||||
// Select all visible text-containing elements
|
||||
const elements = document.querySelectorAll('p, h1, h2, h3, h4, h5, h6, li, td, th, div, span, article, section');
|
||||
elements.forEach(el => {
|
||||
if (el.textContent.trim() && this.isVisible(el) && !this.selectedElements.has(el)) {
|
||||
this.selectElement(el);
|
||||
}
|
||||
});
|
||||
this.updateUI();
|
||||
}
|
||||
}
|
||||
|
||||
isVisible(element) {
|
||||
const rect = element.getBoundingClientRect();
|
||||
const style = window.getComputedStyle(element);
|
||||
return rect.width > 0 &&
|
||||
rect.height > 0 &&
|
||||
style.display !== 'none' &&
|
||||
style.visibility !== 'hidden' &&
|
||||
style.opacity !== '0';
|
||||
}
|
||||
|
||||
selectElement(element) {
|
||||
this.selectedElements.add(element);
|
||||
|
||||
// Create highlight box
|
||||
const box = this.createHighlightBox(element);
|
||||
this.highlightBoxes.set(element, box);
|
||||
|
||||
// Add selected class
|
||||
element.classList.add('c4ai-selected');
|
||||
|
||||
this.selectionCounter++;
|
||||
}
|
||||
|
||||
deselectElement(element) {
|
||||
this.selectedElements.delete(element);
|
||||
|
||||
// Remove highlight box (badge)
|
||||
const badge = this.highlightBoxes.get(element);
|
||||
if (badge) {
|
||||
// Remove scroll/resize listeners
|
||||
if (badge._updatePosition) {
|
||||
window.removeEventListener('scroll', badge._updatePosition, true);
|
||||
window.removeEventListener('resize', badge._updatePosition);
|
||||
}
|
||||
badge.remove();
|
||||
this.highlightBoxes.delete(element);
|
||||
}
|
||||
|
||||
// Remove outline
|
||||
element.style.outline = '';
|
||||
element.style.outlineOffset = '';
|
||||
|
||||
// Remove attributes
|
||||
element.removeAttribute('data-c4ai-selection-order');
|
||||
element.classList.remove('c4ai-selected');
|
||||
|
||||
this.selectionCounter--;
|
||||
}
|
||||
|
||||
createHighlightBox(element) {
|
||||
// Add a data attribute to track selection order
|
||||
element.setAttribute('data-c4ai-selection-order', this.selectionCounter + 1);
|
||||
|
||||
// Add selection outline directly to the element
|
||||
element.style.outline = '2px solid #0fbbaa';
|
||||
element.style.outlineOffset = '2px';
|
||||
|
||||
// Create badge with fixed positioning
|
||||
const badge = document.createElement('div');
|
||||
badge.className = 'c4ai-selection-badge-fixed';
|
||||
badge.textContent = this.selectionCounter + 1;
|
||||
badge.setAttribute('data-c4ai-badge', 'true');
|
||||
badge.title = 'Click to deselect';
|
||||
|
||||
// Get element position and set badge position
|
||||
const rect = element.getBoundingClientRect();
|
||||
badge.style.cssText = `
|
||||
position: fixed !important;
|
||||
top: ${rect.top - 12}px !important;
|
||||
left: ${rect.left - 12}px !important;
|
||||
width: 24px !important;
|
||||
height: 24px !important;
|
||||
background: #0fbbaa !important;
|
||||
color: #070708 !important;
|
||||
border-radius: 50% !important;
|
||||
display: flex !important;
|
||||
align-items: center !important;
|
||||
justify-content: center !important;
|
||||
font-size: 12px !important;
|
||||
font-weight: bold !important;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif !important;
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3) !important;
|
||||
z-index: 999998 !important;
|
||||
cursor: pointer !important;
|
||||
transition: all 0.2s ease !important;
|
||||
pointer-events: auto !important;
|
||||
border: none !important;
|
||||
padding: 0 !important;
|
||||
margin: 0 !important;
|
||||
line-height: 1 !important;
|
||||
text-align: center !important;
|
||||
text-decoration: none !important;
|
||||
box-sizing: border-box !important;
|
||||
`;
|
||||
|
||||
// Add hover styles dynamically
|
||||
badge.addEventListener('mouseenter', () => {
|
||||
badge.style.setProperty('background', '#ff3c74', 'important');
|
||||
badge.style.setProperty('transform', 'scale(1.1)', 'important');
|
||||
});
|
||||
|
||||
badge.addEventListener('mouseleave', () => {
|
||||
badge.style.setProperty('background', '#0fbbaa', 'important');
|
||||
badge.style.setProperty('transform', 'scale(1)', 'important');
|
||||
});
|
||||
|
||||
// Add click handler to badge for deselection
|
||||
badge.addEventListener('click', (e) => {
|
||||
e.stopPropagation();
|
||||
e.preventDefault();
|
||||
this.deselectElement(element);
|
||||
this.updateUI();
|
||||
});
|
||||
|
||||
// Add scroll listener to update position
|
||||
const updatePosition = () => {
|
||||
const newRect = element.getBoundingClientRect();
|
||||
badge.style.top = `${newRect.top - 12}px`;
|
||||
badge.style.left = `${newRect.left - 12}px`;
|
||||
};
|
||||
|
||||
// Store the update function so we can remove it later
|
||||
badge._updatePosition = updatePosition;
|
||||
window.addEventListener('scroll', updatePosition, true);
|
||||
window.addEventListener('resize', updatePosition);
|
||||
|
||||
document.body.appendChild(badge);
|
||||
|
||||
return badge;
|
||||
}
|
||||
|
||||
clearSelection() {
|
||||
// Clear all selections
|
||||
this.selectedElements.forEach(element => {
|
||||
// Remove badge
|
||||
const badge = this.highlightBoxes.get(element);
|
||||
if (badge) {
|
||||
// Remove scroll/resize listeners
|
||||
if (badge._updatePosition) {
|
||||
window.removeEventListener('scroll', badge._updatePosition, true);
|
||||
window.removeEventListener('resize', badge._updatePosition);
|
||||
}
|
||||
badge.remove();
|
||||
}
|
||||
|
||||
// Remove outline
|
||||
element.style.outline = '';
|
||||
element.style.outlineOffset = '';
|
||||
|
||||
// Remove attributes
|
||||
element.removeAttribute('data-c4ai-selection-order');
|
||||
element.classList.remove('c4ai-selected');
|
||||
});
|
||||
|
||||
this.selectedElements.clear();
|
||||
this.highlightBoxes.clear();
|
||||
this.selectionCounter = 0;
|
||||
|
||||
this.updateUI();
|
||||
}
|
||||
|
||||
updateUI() {
|
||||
const count = this.selectedElements.size;
|
||||
|
||||
// Update selection count
|
||||
this.toolbar.querySelector('.c4ai-selection-count').textContent =
|
||||
`${count} element${count !== 1 ? 's' : ''} selected`;
|
||||
|
||||
// Enable/disable buttons
|
||||
const hasSelection = count > 0;
|
||||
this.toolbar.querySelector('.c4ai-preview-btn').disabled = !hasSelection;
|
||||
this.toolbar.querySelector('.c4ai-copy-btn').disabled = !hasSelection;
|
||||
this.toolbar.querySelector('.c4ai-clear-btn').disabled = !hasSelection;
|
||||
}
|
||||
|
||||
async showPreview() {
|
||||
// Initialize markdown preview modal if not already done
|
||||
if (!this.markdownPreviewModal) {
|
||||
this.markdownPreviewModal = new MarkdownPreviewModal();
|
||||
}
|
||||
|
||||
// Show modal with callback to generate markdown
|
||||
this.markdownPreviewModal.show(async (options) => {
|
||||
return await this.generateMarkdown(options);
|
||||
});
|
||||
}
|
||||
|
||||
/* createPreviewPanel() {
|
||||
this.previewPanel = document.createElement('div');
|
||||
this.previewPanel.className = 'c4ai-c2c-preview';
|
||||
this.previewPanel.innerHTML = `
|
||||
<div class="c4ai-preview-header">
|
||||
<div class="c4ai-toolbar-dots">
|
||||
<span class="c4ai-dot c4ai-dot-red"></span>
|
||||
<span class="c4ai-dot c4ai-dot-yellow"></span>
|
||||
<span class="c4ai-dot c4ai-dot-green"></span>
|
||||
</div>
|
||||
<span class="c4ai-preview-title">Markdown Preview</span>
|
||||
<button class="c4ai-preview-close">×</button>
|
||||
</div>
|
||||
<div class="c4ai-preview-options">
|
||||
<label><input type="checkbox" name="textOnly"> 👁️ Visual Text Mode (As You See) TRY THIS!!!</label>
|
||||
<label><input type="checkbox" name="includeImages" checked> Include Images</label>
|
||||
<label><input type="checkbox" name="preserveTables" checked> Preserve Tables</label>
|
||||
<label><input type="checkbox" name="preserveLinks" checked> Preserve Links</label>
|
||||
<label><input type="checkbox" name="keepCodeFormatting" checked> Keep Code Formatting</label>
|
||||
<label><input type="checkbox" name="simplifyLayout"> Simplify Layout</label>
|
||||
<label><input type="checkbox" name="addSeparators" checked> Add Separators</label>
|
||||
<label><input type="checkbox" name="includeXPath"> Include XPath Headers</label>
|
||||
</div>
|
||||
<div class="c4ai-preview-content">
|
||||
<div class="c4ai-preview-tabs">
|
||||
<button class="c4ai-tab active" data-tab="preview">Preview</button>
|
||||
<button class="c4ai-tab" data-tab="markdown">Markdown</button>
|
||||
<button class="c4ai-wrap-toggle" title="Toggle word wrap">↔️ Wrap</button>
|
||||
</div>
|
||||
<div class="c4ai-preview-pane active" data-pane="preview"></div>
|
||||
<div class="c4ai-preview-pane" data-pane="markdown"></div>
|
||||
</div>
|
||||
<div class="c4ai-preview-actions">
|
||||
<button class="c4ai-download-btn">Download .md</button>
|
||||
<button class="c4ai-copy-markdown-btn">Copy Markdown</button>
|
||||
<button class="c4ai-cloud-btn" disabled>Send to Cloud (Coming Soon)</button>
|
||||
</div>
|
||||
`;
|
||||
|
||||
document.body.appendChild(this.previewPanel);
|
||||
makeDraggableByHeader(this.previewPanel);
|
||||
|
||||
// Position preview panel
|
||||
this.previewPanel.style.position = 'fixed';
|
||||
this.previewPanel.style.top = '50%';
|
||||
this.previewPanel.style.left = '50%';
|
||||
this.previewPanel.style.transform = 'translate(-50%, -50%)';
|
||||
this.previewPanel.style.zIndex = '999999';
|
||||
|
||||
this.setupPreviewEventListeners();
|
||||
} */
|
||||
|
||||
/* setupPreviewEventListeners() {
|
||||
// Close button
|
||||
this.previewPanel.querySelector('.c4ai-preview-close').addEventListener('click', () => {
|
||||
this.previewPanel.style.display = 'none';
|
||||
});
|
||||
|
||||
// Tab switching
|
||||
this.previewPanel.querySelectorAll('.c4ai-tab').forEach(tab => {
|
||||
tab.addEventListener('click', (e) => {
|
||||
const tabName = e.target.dataset.tab;
|
||||
this.switchPreviewTab(tabName);
|
||||
});
|
||||
});
|
||||
|
||||
// Wrap toggle
|
||||
const wrapToggle = this.previewPanel.querySelector('.c4ai-wrap-toggle');
|
||||
wrapToggle.addEventListener('click', () => {
|
||||
const panes = this.previewPanel.querySelectorAll('.c4ai-preview-pane');
|
||||
panes.forEach(pane => {
|
||||
pane.classList.toggle('wrap');
|
||||
});
|
||||
wrapToggle.classList.toggle('active');
|
||||
});
|
||||
|
||||
// Options change
|
||||
this.previewPanel.querySelectorAll('input[type="checkbox"]').forEach(checkbox => {
|
||||
checkbox.addEventListener('change', async (e) => {
|
||||
this.options[e.target.name] = e.target.checked;
|
||||
|
||||
// If text-only is enabled, automatically disable certain options
|
||||
if (e.target.name === 'textOnly' && e.target.checked) {
|
||||
// Update UI checkboxes
|
||||
const preserveLinksCheckbox = this.previewPanel.querySelector('input[name="preserveLinks"]');
|
||||
if (preserveLinksCheckbox) {
|
||||
preserveLinksCheckbox.checked = false;
|
||||
preserveLinksCheckbox.disabled = true;
|
||||
}
|
||||
|
||||
// Optionally disable images in text-only mode
|
||||
const includeImagesCheckbox = this.previewPanel.querySelector('input[name="includeImages"]');
|
||||
if (includeImagesCheckbox) {
|
||||
includeImagesCheckbox.disabled = true;
|
||||
}
|
||||
} else if (e.target.name === 'textOnly' && !e.target.checked) {
|
||||
// Re-enable options when text-only is disabled
|
||||
const preserveLinksCheckbox = this.previewPanel.querySelector('input[name="preserveLinks"]');
|
||||
if (preserveLinksCheckbox) {
|
||||
preserveLinksCheckbox.disabled = false;
|
||||
}
|
||||
|
||||
const includeImagesCheckbox = this.previewPanel.querySelector('input[name="includeImages"]');
|
||||
if (includeImagesCheckbox) {
|
||||
includeImagesCheckbox.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
const markdown = await this.generateMarkdown();
|
||||
await this.updatePreviewContent(markdown);
|
||||
});
|
||||
});
|
||||
|
||||
// Action buttons
|
||||
this.previewPanel.querySelector('.c4ai-copy-markdown-btn').addEventListener('click', () => {
|
||||
this.copyToClipboard();
|
||||
});
|
||||
|
||||
this.previewPanel.querySelector('.c4ai-download-btn').addEventListener('click', () => {
|
||||
this.downloadMarkdown();
|
||||
});
|
||||
} */
|
||||
|
||||
/* switchPreviewTab(tabName) {
|
||||
// Update active tab
|
||||
this.previewPanel.querySelectorAll('.c4ai-tab').forEach(tab => {
|
||||
tab.classList.toggle('active', tab.dataset.tab === tabName);
|
||||
});
|
||||
|
||||
// Update active pane
|
||||
this.previewPanel.querySelectorAll('.c4ai-preview-pane').forEach(pane => {
|
||||
pane.classList.toggle('active', pane.dataset.pane === tabName);
|
||||
});
|
||||
} */
|
||||
|
||||
/* async updatePreviewContent(markdown) {
|
||||
// Update markdown pane
|
||||
const markdownPane = this.previewPanel.querySelector('[data-pane="markdown"]');
|
||||
markdownPane.innerHTML = `<pre><code>${this.escapeHtml(markdown)}</code></pre>`;
|
||||
|
||||
// Update preview pane using marked.js
|
||||
const previewPane = this.previewPanel.querySelector('[data-pane="preview"]');
|
||||
|
||||
// Configure marked options (marked.js is already loaded via manifest)
|
||||
if (window.marked) {
|
||||
marked.setOptions({
|
||||
gfm: true,
|
||||
breaks: true,
|
||||
tables: true,
|
||||
headerIds: false,
|
||||
mangle: false
|
||||
});
|
||||
|
||||
// Render markdown to HTML
|
||||
const html = marked.parse(markdown);
|
||||
previewPane.innerHTML = `<div class="c4ai-markdown-preview">${html}</div>`;
|
||||
} else {
|
||||
// Fallback if marked.js is not available
|
||||
previewPane.innerHTML = `<div class="c4ai-markdown-preview"><pre>${this.escapeHtml(markdown)}</pre></div>`;
|
||||
}
|
||||
} */
|
||||
|
||||
|
||||
/* escapeHtml(unsafe) {
|
||||
return unsafe
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
} */
|
||||
|
||||
async generateMarkdown(options) {
|
||||
// Get selected elements as array
|
||||
const elements = Array.from(this.selectedElements);
|
||||
|
||||
// Sort elements by their selection order
|
||||
const sortedElements = elements.sort((a, b) => {
|
||||
const orderA = parseInt(a.getAttribute('data-c4ai-selection-order') || '0');
|
||||
const orderB = parseInt(b.getAttribute('data-c4ai-selection-order') || '0');
|
||||
return orderA - orderB;
|
||||
});
|
||||
|
||||
// Convert each element separately
|
||||
const markdownParts = [];
|
||||
|
||||
for (let i = 0; i < sortedElements.length; i++) {
|
||||
const element = sortedElements[i];
|
||||
|
||||
// Add XPath header if enabled
|
||||
if (options.includeXPath) {
|
||||
const xpath = this.getXPath(element);
|
||||
markdownParts.push(`### Element ${i + 1} - XPath: \`${xpath}\`\n`);
|
||||
}
|
||||
|
||||
// Check if element is part of a table structure that should be processed specially
|
||||
let elementsToConvert = [element];
|
||||
|
||||
// If text-only mode and element is a TR, process the entire table for better context
|
||||
if (options.textOnly && element.tagName === 'TR') {
|
||||
const table = element.closest('table');
|
||||
if (table && !sortedElements.includes(table)) {
|
||||
// Only include this table row, not the whole table
|
||||
elementsToConvert = [element];
|
||||
}
|
||||
}
|
||||
|
||||
// Analyze and convert individual element
|
||||
const analysis = await this.contentAnalyzer.analyze(elementsToConvert);
|
||||
const markdown = await this.markdownConverter.convert(elementsToConvert, {
|
||||
...options,
|
||||
analysis
|
||||
});
|
||||
|
||||
// Trim the markdown before adding
|
||||
const trimmedMarkdown = markdown.trim();
|
||||
markdownParts.push(trimmedMarkdown);
|
||||
|
||||
// Add separator if enabled and not last element
|
||||
if (options.addSeparators && i < sortedElements.length - 1) {
|
||||
markdownParts.push('\n---\n');
|
||||
}
|
||||
}
|
||||
|
||||
return markdownParts.join('\n');
|
||||
}
|
||||
|
||||
getXPath(element) {
|
||||
if (element.id) {
|
||||
return `//*[@id="${element.id}"]`;
|
||||
}
|
||||
|
||||
const parts = [];
|
||||
let current = element;
|
||||
|
||||
while (current && current.nodeType === Node.ELEMENT_NODE) {
|
||||
let index = 0;
|
||||
let sibling = current.previousSibling;
|
||||
|
||||
while (sibling) {
|
||||
if (sibling.nodeType === Node.ELEMENT_NODE && sibling.nodeName === current.nodeName) {
|
||||
index++;
|
||||
}
|
||||
sibling = sibling.previousSibling;
|
||||
}
|
||||
|
||||
const tagName = current.nodeName.toLowerCase();
|
||||
const part = index > 0 ? `${tagName}[${index + 1}]` : tagName;
|
||||
parts.unshift(part);
|
||||
|
||||
current = current.parentNode;
|
||||
}
|
||||
|
||||
return '/' + parts.join('/');
|
||||
}
|
||||
|
||||
sortElementsByPosition(elements) {
|
||||
return elements.sort((a, b) => {
|
||||
const position = a.compareDocumentPosition(b);
|
||||
if (position & Node.DOCUMENT_POSITION_FOLLOWING) {
|
||||
return -1;
|
||||
} else if (position & Node.DOCUMENT_POSITION_PRECEDING) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
async copyToClipboard() {
|
||||
if (this.markdownPreviewModal) {
|
||||
await this.markdownPreviewModal.copyToClipboard();
|
||||
}
|
||||
}
|
||||
|
||||
async downloadMarkdown() {
|
||||
if (this.markdownPreviewModal) {
|
||||
await this.markdownPreviewModal.downloadMarkdown();
|
||||
}
|
||||
}
|
||||
|
||||
showNotification(message, type = 'success') {
|
||||
const notification = document.createElement('div');
|
||||
notification.className = `c4ai-notification c4ai-notification-${type}`;
|
||||
notification.textContent = message;
|
||||
|
||||
document.body.appendChild(notification);
|
||||
|
||||
// Animate in
|
||||
setTimeout(() => notification.classList.add('show'), 10);
|
||||
|
||||
// Remove after 3 seconds
|
||||
setTimeout(() => {
|
||||
notification.classList.remove('show');
|
||||
setTimeout(() => notification.remove(), 300);
|
||||
}, 3000);
|
||||
}
|
||||
|
||||
deactivate() {
|
||||
// Remove event listeners
|
||||
document.removeEventListener('click', this.documentClickHandler, true);
|
||||
document.removeEventListener('click', this.linkClickHandler, true);
|
||||
document.removeEventListener('mouseover', this.documentHoverHandler, true);
|
||||
document.removeEventListener('mouseout', this.documentMouseOutHandler, true);
|
||||
document.removeEventListener('keydown', this.keyboardHandler);
|
||||
|
||||
// Clear selections
|
||||
this.clearSelection();
|
||||
|
||||
// Remove UI elements
|
||||
if (this.toolbar) {
|
||||
this.toolbar.remove();
|
||||
this.toolbar = null;
|
||||
}
|
||||
|
||||
if (this.markdownPreviewModal) {
|
||||
this.markdownPreviewModal.destroy();
|
||||
this.markdownPreviewModal = null;
|
||||
}
|
||||
|
||||
// Remove hover styles
|
||||
document.querySelectorAll('.c4ai-hover-candidate').forEach(el => {
|
||||
el.classList.remove('c4ai-hover-candidate');
|
||||
});
|
||||
|
||||
// Notify background script (with error handling)
|
||||
try {
|
||||
if (chrome.runtime && chrome.runtime.sendMessage) {
|
||||
chrome.runtime.sendMessage({
|
||||
action: 'c2cDeactivated'
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
// Extension context might be invalidated, ignore the error
|
||||
console.log('Markdown Extraction deactivated (extension context unavailable)');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
// Shared Markdown Preview Modal Component for Crawl4AI Assistant
|
||||
// Used by both SchemaBuilder and Click2CrawlBuilder
|
||||
|
||||
class MarkdownPreviewModal {
|
||||
constructor(options = {}) {
|
||||
this.modal = null;
|
||||
this.markdownOptions = {
|
||||
includeImages: true,
|
||||
preserveTables: true,
|
||||
keepCodeFormatting: true,
|
||||
simplifyLayout: false,
|
||||
preserveLinks: true,
|
||||
addSeparators: true,
|
||||
includeXPath: false,
|
||||
textOnly: false,
|
||||
...options
|
||||
};
|
||||
this.onGenerateMarkdown = null;
|
||||
this.currentMarkdown = '';
|
||||
}
|
||||
|
||||
show(generateMarkdownCallback) {
|
||||
this.onGenerateMarkdown = generateMarkdownCallback;
|
||||
|
||||
if (!this.modal) {
|
||||
this.createModal();
|
||||
}
|
||||
|
||||
// Generate initial markdown
|
||||
this.updateContent();
|
||||
this.modal.style.display = 'block';
|
||||
}
|
||||
|
||||
hide() {
|
||||
if (this.modal) {
|
||||
this.modal.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
createModal() {
|
||||
this.modal = document.createElement('div');
|
||||
this.modal.className = 'c4ai-c2c-preview';
|
||||
this.modal.innerHTML = `
|
||||
<div class="c4ai-preview-header">
|
||||
<div class="c4ai-toolbar-dots">
|
||||
<span class="c4ai-dot c4ai-dot-red"></span>
|
||||
<span class="c4ai-dot c4ai-dot-yellow"></span>
|
||||
<span class="c4ai-dot c4ai-dot-green"></span>
|
||||
</div>
|
||||
<span class="c4ai-preview-title">Markdown Preview</span>
|
||||
<button class="c4ai-preview-close">×</button>
|
||||
</div>
|
||||
<div class="c4ai-preview-options">
|
||||
<label><input type="checkbox" name="textOnly"> 👁️ Visual Text Mode (As You See)</label>
|
||||
<label><input type="checkbox" name="includeImages" checked> Include Images</label>
|
||||
<label><input type="checkbox" name="preserveTables" checked> Preserve Tables</label>
|
||||
<label><input type="checkbox" name="preserveLinks" checked> Preserve Links</label>
|
||||
<label><input type="checkbox" name="keepCodeFormatting" checked> Keep Code Formatting</label>
|
||||
<label><input type="checkbox" name="simplifyLayout"> Simplify Layout</label>
|
||||
<label><input type="checkbox" name="addSeparators" checked> Add Separators</label>
|
||||
<label><input type="checkbox" name="includeXPath"> Include XPath Headers</label>
|
||||
</div>
|
||||
<div class="c4ai-preview-content">
|
||||
<div class="c4ai-preview-tabs">
|
||||
<button class="c4ai-tab active" data-tab="preview">Preview</button>
|
||||
<button class="c4ai-tab" data-tab="markdown">Markdown</button>
|
||||
<button class="c4ai-wrap-toggle" title="Toggle word wrap">↔️ Wrap</button>
|
||||
</div>
|
||||
<div class="c4ai-preview-pane active" data-pane="preview"></div>
|
||||
<div class="c4ai-preview-pane" data-pane="markdown"></div>
|
||||
</div>
|
||||
<div class="c4ai-preview-actions">
|
||||
<button class="c4ai-download-btn">Download .md</button>
|
||||
<button class="c4ai-copy-markdown-btn">Copy Markdown</button>
|
||||
<button class="c4ai-cloud-btn" disabled>Send to Cloud (Coming Soon)</button>
|
||||
</div>
|
||||
`;
|
||||
|
||||
document.body.appendChild(this.modal);
|
||||
|
||||
// Make modal draggable
|
||||
if (window.C4AI_Utils && window.C4AI_Utils.makeDraggable) {
|
||||
window.C4AI_Utils.makeDraggable(this.modal);
|
||||
}
|
||||
|
||||
// Position preview modal
|
||||
this.modal.style.position = 'fixed';
|
||||
this.modal.style.top = '50%';
|
||||
this.modal.style.left = '50%';
|
||||
this.modal.style.transform = 'translate(-50%, -50%)';
|
||||
this.modal.style.zIndex = '999999';
|
||||
|
||||
this.setupEventListeners();
|
||||
}
|
||||
|
||||
setupEventListeners() {
|
||||
// Close button
|
||||
this.modal.querySelector('.c4ai-preview-close').addEventListener('click', () => {
|
||||
this.hide();
|
||||
});
|
||||
|
||||
// Tab switching
|
||||
this.modal.querySelectorAll('.c4ai-tab').forEach(tab => {
|
||||
tab.addEventListener('click', (e) => {
|
||||
const tabName = e.target.dataset.tab;
|
||||
this.switchTab(tabName);
|
||||
});
|
||||
});
|
||||
|
||||
// Wrap toggle
|
||||
const wrapToggle = this.modal.querySelector('.c4ai-wrap-toggle');
|
||||
wrapToggle.addEventListener('click', () => {
|
||||
const panes = this.modal.querySelectorAll('.c4ai-preview-pane');
|
||||
panes.forEach(pane => {
|
||||
pane.classList.toggle('wrap');
|
||||
});
|
||||
wrapToggle.classList.toggle('active');
|
||||
});
|
||||
|
||||
// Options change
|
||||
this.modal.querySelectorAll('input[type="checkbox"]').forEach(checkbox => {
|
||||
checkbox.addEventListener('change', async (e) => {
|
||||
this.markdownOptions[e.target.name] = e.target.checked;
|
||||
|
||||
// Handle text-only mode dependencies
|
||||
if (e.target.name === 'textOnly' && e.target.checked) {
|
||||
const preserveLinksCheckbox = this.modal.querySelector('input[name="preserveLinks"]');
|
||||
if (preserveLinksCheckbox) {
|
||||
preserveLinksCheckbox.checked = false;
|
||||
preserveLinksCheckbox.disabled = true;
|
||||
this.markdownOptions.preserveLinks = false;
|
||||
}
|
||||
|
||||
const includeImagesCheckbox = this.modal.querySelector('input[name="includeImages"]');
|
||||
if (includeImagesCheckbox) {
|
||||
includeImagesCheckbox.disabled = true;
|
||||
}
|
||||
} else if (e.target.name === 'textOnly' && !e.target.checked) {
|
||||
// Re-enable options when text-only is disabled
|
||||
const preserveLinksCheckbox = this.modal.querySelector('input[name="preserveLinks"]');
|
||||
if (preserveLinksCheckbox) {
|
||||
preserveLinksCheckbox.disabled = false;
|
||||
}
|
||||
|
||||
const includeImagesCheckbox = this.modal.querySelector('input[name="includeImages"]');
|
||||
if (includeImagesCheckbox) {
|
||||
includeImagesCheckbox.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Update markdown content
|
||||
await this.updateContent();
|
||||
});
|
||||
});
|
||||
|
||||
// Action buttons
|
||||
this.modal.querySelector('.c4ai-copy-markdown-btn').addEventListener('click', () => {
|
||||
this.copyToClipboard();
|
||||
});
|
||||
|
||||
this.modal.querySelector('.c4ai-download-btn').addEventListener('click', () => {
|
||||
this.downloadMarkdown();
|
||||
});
|
||||
}
|
||||
|
||||
switchTab(tabName) {
|
||||
// Update active tab
|
||||
this.modal.querySelectorAll('.c4ai-tab').forEach(tab => {
|
||||
tab.classList.toggle('active', tab.dataset.tab === tabName);
|
||||
});
|
||||
|
||||
// Update active pane
|
||||
this.modal.querySelectorAll('.c4ai-preview-pane').forEach(pane => {
|
||||
pane.classList.toggle('active', pane.dataset.pane === tabName);
|
||||
});
|
||||
}
|
||||
|
||||
async updateContent() {
|
||||
if (!this.onGenerateMarkdown) return;
|
||||
|
||||
try {
|
||||
// Generate markdown with current options
|
||||
this.currentMarkdown = await this.onGenerateMarkdown(this.markdownOptions);
|
||||
|
||||
// Update markdown pane
|
||||
const markdownPane = this.modal.querySelector('[data-pane="markdown"]');
|
||||
markdownPane.innerHTML = `<pre><code>${this.escapeHtml(this.currentMarkdown)}</code></pre>`;
|
||||
|
||||
// Update preview pane
|
||||
const previewPane = this.modal.querySelector('[data-pane="preview"]');
|
||||
|
||||
// Use marked.js if available
|
||||
if (window.marked) {
|
||||
marked.setOptions({
|
||||
gfm: true,
|
||||
breaks: true,
|
||||
tables: true,
|
||||
headerIds: false,
|
||||
mangle: false
|
||||
});
|
||||
|
||||
const html = marked.parse(this.currentMarkdown);
|
||||
previewPane.innerHTML = `<div class="c4ai-markdown-preview">${html}</div>`;
|
||||
} else {
|
||||
// Fallback
|
||||
previewPane.innerHTML = `<div class="c4ai-markdown-preview"><pre>${this.escapeHtml(this.currentMarkdown)}</pre></div>`;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error generating markdown:', error);
|
||||
this.showNotification('Error generating markdown', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
async copyToClipboard() {
|
||||
try {
|
||||
await navigator.clipboard.writeText(this.currentMarkdown);
|
||||
this.showNotification('Markdown copied to clipboard!');
|
||||
} catch (err) {
|
||||
console.error('Failed to copy:', err);
|
||||
this.showNotification('Failed to copy. Please try again.', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
async downloadMarkdown() {
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, -5);
|
||||
const filename = `crawl4ai-export-${timestamp}.md`;
|
||||
|
||||
// Create blob and download
|
||||
const blob = new Blob([this.currentMarkdown], { type: 'text/markdown' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
|
||||
this.showNotification(`Downloaded ${filename}`);
|
||||
}
|
||||
|
||||
showNotification(message, type = 'success') {
|
||||
const notification = document.createElement('div');
|
||||
notification.className = `c4ai-notification c4ai-notification-${type}`;
|
||||
notification.textContent = message;
|
||||
|
||||
document.body.appendChild(notification);
|
||||
|
||||
// Animate in
|
||||
setTimeout(() => notification.classList.add('show'), 10);
|
||||
|
||||
// Remove after 3 seconds
|
||||
setTimeout(() => {
|
||||
notification.classList.remove('show');
|
||||
setTimeout(() => notification.remove(), 300);
|
||||
}, 3000);
|
||||
}
|
||||
|
||||
escapeHtml(unsafe) {
|
||||
return unsafe
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
// Get current options
|
||||
getOptions() {
|
||||
return { ...this.markdownOptions };
|
||||
}
|
||||
|
||||
// Update options programmatically
|
||||
setOptions(options) {
|
||||
this.markdownOptions = { ...this.markdownOptions, ...options };
|
||||
|
||||
// Update checkboxes to reflect new options
|
||||
Object.entries(options).forEach(([key, value]) => {
|
||||
const checkbox = this.modal?.querySelector(`input[name="${key}"]`);
|
||||
if (checkbox && typeof value === 'boolean') {
|
||||
checkbox.checked = value;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
destroy() {
|
||||
if (this.modal) {
|
||||
this.modal.remove();
|
||||
this.modal = null;
|
||||
}
|
||||
this.onGenerateMarkdown = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Export for use in other scripts
|
||||
if (typeof window !== 'undefined') {
|
||||
window.MarkdownPreviewModal = MarkdownPreviewModal;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,253 @@
|
||||
// Shared utilities for Crawl4AI Chrome Extension
|
||||
|
||||
// Make element draggable by its titlebar
|
||||
function makeDraggable(element) {
|
||||
let isDragging = false;
|
||||
let startX, startY, initialX, initialY;
|
||||
|
||||
const titlebar = element.querySelector('.c4ai-toolbar-titlebar, .c4ai-titlebar');
|
||||
if (!titlebar) return;
|
||||
|
||||
titlebar.addEventListener('mousedown', (e) => {
|
||||
// Don't drag if clicking on buttons
|
||||
if (e.target.classList.contains('c4ai-dot') || e.target.closest('button')) return;
|
||||
|
||||
isDragging = true;
|
||||
startX = e.clientX;
|
||||
startY = e.clientY;
|
||||
|
||||
const rect = element.getBoundingClientRect();
|
||||
initialX = rect.left;
|
||||
initialY = rect.top;
|
||||
|
||||
element.style.transition = 'none';
|
||||
titlebar.style.cursor = 'grabbing';
|
||||
});
|
||||
|
||||
document.addEventListener('mousemove', (e) => {
|
||||
if (!isDragging) return;
|
||||
|
||||
const deltaX = e.clientX - startX;
|
||||
const deltaY = e.clientY - startY;
|
||||
|
||||
element.style.left = `${initialX + deltaX}px`;
|
||||
element.style.top = `${initialY + deltaY}px`;
|
||||
element.style.right = 'auto';
|
||||
});
|
||||
|
||||
document.addEventListener('mouseup', () => {
|
||||
if (isDragging) {
|
||||
isDragging = false;
|
||||
element.style.transition = '';
|
||||
titlebar.style.cursor = 'grab';
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Make element draggable by a specific header element
|
||||
function makeDraggableByHeader(element) {
|
||||
let isDragging = false;
|
||||
let startX, startY, initialX, initialY;
|
||||
|
||||
const header = element.querySelector('.c4ai-debugger-header');
|
||||
if (!header) return;
|
||||
|
||||
header.addEventListener('mousedown', (e) => {
|
||||
// Don't drag if clicking on close button
|
||||
if (e.target.id === 'c4ai-close-debugger' || e.target.closest('#c4ai-close-debugger')) return;
|
||||
|
||||
isDragging = true;
|
||||
startX = e.clientX;
|
||||
startY = e.clientY;
|
||||
|
||||
const rect = element.getBoundingClientRect();
|
||||
initialX = rect.left;
|
||||
initialY = rect.top;
|
||||
|
||||
element.style.transition = 'none';
|
||||
header.style.cursor = 'grabbing';
|
||||
});
|
||||
|
||||
document.addEventListener('mousemove', (e) => {
|
||||
if (!isDragging) return;
|
||||
|
||||
const deltaX = e.clientX - startX;
|
||||
const deltaY = e.clientY - startY;
|
||||
|
||||
element.style.left = `${initialX + deltaX}px`;
|
||||
element.style.top = `${initialY + deltaY}px`;
|
||||
element.style.right = 'auto';
|
||||
});
|
||||
|
||||
document.addEventListener('mouseup', () => {
|
||||
if (isDragging) {
|
||||
isDragging = false;
|
||||
element.style.transition = '';
|
||||
header.style.cursor = 'grab';
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Escape HTML for safe display
|
||||
function escapeHtml(text) {
|
||||
const div = document.createElement('div');
|
||||
div.textContent = text;
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
// Apply syntax highlighting to Python code
|
||||
function applySyntaxHighlighting(codeElement) {
|
||||
const code = codeElement.textContent;
|
||||
|
||||
// Split by lines to handle line-by-line
|
||||
const lines = code.split('\n');
|
||||
const highlightedLines = lines.map(line => {
|
||||
let highlightedLine = escapeHtml(line);
|
||||
|
||||
// Skip if line is empty
|
||||
if (!highlightedLine.trim()) return highlightedLine;
|
||||
|
||||
// Comments (lines starting with #)
|
||||
if (highlightedLine.trim().startsWith('#')) {
|
||||
return `<span class="c4ai-comment">${highlightedLine}</span>`;
|
||||
}
|
||||
|
||||
// Triple quoted strings
|
||||
if (highlightedLine.includes('"""')) {
|
||||
highlightedLine = highlightedLine.replace(/(""".*?""")/g, '<span class="c4ai-string">$1</span>');
|
||||
}
|
||||
|
||||
// Regular strings - single and double quotes
|
||||
highlightedLine = highlightedLine.replace(/(["'])([^"']*)\1/g, '<span class="c4ai-string">$1$2$1</span>');
|
||||
|
||||
// Keywords - only highlight if not inside a string
|
||||
const keywords = ['import', 'from', 'async', 'def', 'await', 'try', 'except', 'with', 'as', 'for', 'if', 'else', 'elif', 'return', 'print', 'open', 'and', 'or', 'not', 'in', 'is', 'class', 'self', 'None', 'True', 'False', '__name__', '__main__'];
|
||||
|
||||
keywords.forEach(keyword => {
|
||||
// Use word boundaries and lookahead/lookbehind to ensure we're not in a string
|
||||
const regex = new RegExp(`\\b(${keyword})\\b(?![^<]*</span>)`, 'g');
|
||||
highlightedLine = highlightedLine.replace(regex, '<span class="c4ai-keyword">$1</span>');
|
||||
});
|
||||
|
||||
// Functions (word followed by parenthesis)
|
||||
highlightedLine = highlightedLine.replace(/\b([a-zA-Z_]\w*)\s*\(/g, '<span class="c4ai-function">$1</span>(');
|
||||
|
||||
return highlightedLine;
|
||||
});
|
||||
|
||||
codeElement.innerHTML = highlightedLines.join('\n');
|
||||
}
|
||||
|
||||
// Apply syntax highlighting to JavaScript code
|
||||
function applySyntaxHighlightingJS(codeElement) {
|
||||
const code = codeElement.textContent;
|
||||
|
||||
// Split by lines to handle line-by-line
|
||||
const lines = code.split('\n');
|
||||
const highlightedLines = lines.map(line => {
|
||||
let highlightedLine = escapeHtml(line);
|
||||
|
||||
// Skip if line is empty
|
||||
if (!highlightedLine.trim()) return highlightedLine;
|
||||
|
||||
// Comments
|
||||
if (highlightedLine.trim().startsWith('//')) {
|
||||
return `<span class="c4ai-comment">${highlightedLine}</span>`;
|
||||
}
|
||||
|
||||
// Multi-line comments
|
||||
highlightedLine = highlightedLine.replace(/(\/\*.*?\*\/)/g, '<span class="c4ai-comment">$1</span>');
|
||||
|
||||
// Template literals
|
||||
highlightedLine = highlightedLine.replace(/(`[^`]*`)/g, '<span class="c4ai-string">$1</span>');
|
||||
|
||||
// Regular strings - single and double quotes
|
||||
highlightedLine = highlightedLine.replace(/(["'])([^"']*)\1/g, '<span class="c4ai-string">$1$2$1</span>');
|
||||
|
||||
// Keywords
|
||||
const keywords = ['const', 'let', 'var', 'function', 'async', 'await', 'if', 'else', 'for', 'while', 'do', 'switch', 'case', 'break', 'continue', 'return', 'try', 'catch', 'finally', 'throw', 'new', 'this', 'class', 'extends', 'import', 'export', 'default', 'from', 'null', 'undefined', 'true', 'false'];
|
||||
|
||||
keywords.forEach(keyword => {
|
||||
const regex = new RegExp(`\\b(${keyword})\\b(?![^<]*</span>)`, 'g');
|
||||
highlightedLine = highlightedLine.replace(regex, '<span class="c4ai-keyword">$1</span>');
|
||||
});
|
||||
|
||||
// Functions and methods
|
||||
highlightedLine = highlightedLine.replace(/\b([a-zA-Z_$][\w$]*)\s*\(/g, '<span class="c4ai-function">$1</span>(');
|
||||
|
||||
// Numbers
|
||||
highlightedLine = highlightedLine.replace(/\b(\d+)\b/g, '<span class="c4ai-number">$1</span>');
|
||||
|
||||
return highlightedLine;
|
||||
});
|
||||
|
||||
codeElement.innerHTML = highlightedLines.join('\n');
|
||||
}
|
||||
|
||||
// Get element selector
|
||||
function getElementSelector(element) {
|
||||
// Priority: ID > unique class > tag with position
|
||||
if (element.id) {
|
||||
return `#${element.id}`;
|
||||
}
|
||||
|
||||
if (element.className && typeof element.className === 'string') {
|
||||
const classes = element.className.split(' ').filter(c => c && !c.startsWith('c4ai-'));
|
||||
if (classes.length > 0) {
|
||||
const selector = `.${classes[0]}`;
|
||||
if (document.querySelectorAll(selector).length === 1) {
|
||||
return selector;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build a path selector
|
||||
const path = [];
|
||||
let current = element;
|
||||
|
||||
while (current && current !== document.body) {
|
||||
const tagName = current.tagName.toLowerCase();
|
||||
const parent = current.parentElement;
|
||||
|
||||
if (parent) {
|
||||
const siblings = Array.from(parent.children);
|
||||
const index = siblings.indexOf(current) + 1;
|
||||
|
||||
if (siblings.filter(s => s.tagName === current.tagName).length > 1) {
|
||||
path.unshift(`${tagName}:nth-child(${index})`);
|
||||
} else {
|
||||
path.unshift(tagName);
|
||||
}
|
||||
} else {
|
||||
path.unshift(tagName);
|
||||
}
|
||||
|
||||
current = parent;
|
||||
}
|
||||
|
||||
return path.join(' > ');
|
||||
}
|
||||
|
||||
// Check if element is part of our extension UI
|
||||
function isOurElement(element) {
|
||||
return element.classList.contains('c4ai-highlight-box') ||
|
||||
element.classList.contains('c4ai-toolbar') ||
|
||||
element.closest('.c4ai-toolbar') ||
|
||||
element.classList.contains('c4ai-script-toolbar') ||
|
||||
element.closest('.c4ai-script-toolbar') ||
|
||||
element.closest('.c4ai-field-dialog') ||
|
||||
element.closest('.c4ai-code-modal') ||
|
||||
element.closest('.c4ai-wait-dialog') ||
|
||||
element.closest('.c4ai-timeline-modal');
|
||||
}
|
||||
|
||||
// Export utilities
|
||||
window.C4AI_Utils = {
|
||||
makeDraggable,
|
||||
makeDraggableByHeader,
|
||||
escapeHtml,
|
||||
applySyntaxHighlighting,
|
||||
applySyntaxHighlightingJS,
|
||||
getElementSelector,
|
||||
isOurElement
|
||||
};
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 3.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
@@ -0,0 +1,974 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Crawl4AI Assistant - Chrome Extension for Visual Web Scraping</title>
|
||||
<link rel="stylesheet" href="assistant.css">
|
||||
</head>
|
||||
<body>
|
||||
<div class="terminal-container">
|
||||
<div class="header">
|
||||
<div class="header-content">
|
||||
<div class="logo-section">
|
||||
<img src="../../img/favicon-32x32.png" alt="Crawl4AI Logo" class="logo">
|
||||
<div>
|
||||
<h1>Crawl4AI Assistant</h1>
|
||||
<p class="tagline">Chrome Extension for Visual Web Scraping</p>
|
||||
</div>
|
||||
</div>
|
||||
<nav class="nav-links">
|
||||
<a href="../../" class="nav-link">← Back to Docs</a>
|
||||
<a href="../" class="nav-link">All Apps</a>
|
||||
<a href="https://github.com/unclecode/crawl4ai" class="nav-link" target="_blank">GitHub</a>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
<!-- Video Section -->
|
||||
<section class="video-section">
|
||||
<div class="video-wrapper">
|
||||
<video autoplay loop muted playsinline class="demo-video">
|
||||
<source src="demo.mp4" type="video/mp4">
|
||||
Your browser does not support the video tag.
|
||||
</video>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Cloud Announcement Banner -->
|
||||
<section class="cloud-banner-section">
|
||||
<div class="cloud-banner">
|
||||
<div class="cloud-banner-content">
|
||||
<div class="cloud-banner-text">
|
||||
<h3>You don't need Puppeteer. You need Crawl4AI Cloud.</h3>
|
||||
<p>One API call. JS-rendered. No browser cluster to maintain.</p>
|
||||
</div>
|
||||
<button class="cloud-banner-btn" id="joinWaitlistBanner">
|
||||
Get API Key →
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Introduction -->
|
||||
<section class="intro-section">
|
||||
<div class="terminal-window">
|
||||
<div class="terminal-header">
|
||||
<span class="terminal-title">About Crawl4AI Assistant</span>
|
||||
</div>
|
||||
<div class="terminal-content">
|
||||
<p>Transform any website into structured data with just a few clicks! The Crawl4AI Assistant Chrome Extension provides three powerful tools for web scraping and data extraction.</p>
|
||||
|
||||
<div style="background: #0fbbaa; color: #070708; padding: 12px 16px; border-radius: 8px; margin: 16px 0; font-weight: 600;">
|
||||
🎉 NEW: Click2Crawl extracts data INSTANTLY without any LLM! Test your schema and see JSON results immediately in the browser!
|
||||
</div>
|
||||
|
||||
<div class="features-grid">
|
||||
<div class="feature-card">
|
||||
<span class="feature-icon">🎯</span>
|
||||
<h3>Click2Crawl</h3>
|
||||
<p>Visual data extraction - click elements to build schemas instantly!</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<span class="feature-icon">🔴</span>
|
||||
<h3>Script Builder <span style="color: #f380f5; font-size: 0.75rem;">(Alpha)</span></h3>
|
||||
<p>Record browser actions to create automation scripts</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<span class="feature-icon">📝</span>
|
||||
<h3>Markdown Extraction <span style="color: #0fbbaa; font-size: 0.75rem;">(New!)</span></h3>
|
||||
<p>Convert any webpage content to clean markdown with Visual Text Mode</p>
|
||||
</div>
|
||||
<!-- <div class="feature-card">
|
||||
<span class="feature-icon">🐍</span>
|
||||
<h3>Python Code</h3>
|
||||
<p>Get production-ready Crawl4AI code instantly</p>
|
||||
</div> -->
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Quick Start -->
|
||||
<section class="quickstart-section">
|
||||
<h2>Quick Start</h2>
|
||||
<div class="terminal-window">
|
||||
<div class="terminal-header">
|
||||
<span class="terminal-title">Installation</span>
|
||||
</div>
|
||||
<div class="terminal-content">
|
||||
<div class="installation-steps">
|
||||
<div class="step">
|
||||
<span class="step-number">1</span>
|
||||
<div class="step-content">
|
||||
<h4>Download the Extension</h4>
|
||||
<p>Get the latest release from GitHub or use the button below</p>
|
||||
<a href="crawl4ai-assistant-v1.3.0.zip" class="download-button" download>
|
||||
<span class="button-icon">↓</span>
|
||||
Download Extension (v1.3.0)
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
<div class="step">
|
||||
<span class="step-number">2</span>
|
||||
<div class="step-content">
|
||||
<h4>Load in Chrome</h4>
|
||||
<p>Navigate to <code>chrome://extensions/</code> and enable Developer Mode</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="step">
|
||||
<span class="step-number">3</span>
|
||||
<div class="step-content">
|
||||
<h4>Load Unpacked</h4>
|
||||
<p>Click "Load unpacked" and select the extracted extension folder</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Interactive Tools Section -->
|
||||
<section class="interactive-tools">
|
||||
<h2>Explore Our Tools</h2>
|
||||
|
||||
<div class="tools-container">
|
||||
<!-- Left Panel - Tool Selector -->
|
||||
<div class="tools-panel">
|
||||
<div class="tool-selector active" data-tool="click2crawl">
|
||||
<div class="tool-icon">🎯</div>
|
||||
<div class="tool-info">
|
||||
<h3>Click2Crawl</h3>
|
||||
<p>Visual data extraction</p>
|
||||
</div>
|
||||
<div class="tool-status">Available</div>
|
||||
</div>
|
||||
|
||||
<div class="tool-selector" data-tool="script-builder">
|
||||
<div class="tool-icon">🔴</div>
|
||||
<div class="tool-info">
|
||||
<h3>Script Builder</h3>
|
||||
<p>Browser automation</p>
|
||||
</div>
|
||||
<div class="tool-status alpha">Alpha</div>
|
||||
</div>
|
||||
|
||||
<div class="tool-selector" data-tool="markdown-extraction">
|
||||
<div class="tool-icon">📝</div>
|
||||
<div class="tool-info">
|
||||
<h3>Markdown Extraction</h3>
|
||||
<p>Content to markdown</p>
|
||||
</div>
|
||||
<div class="tool-status new">New!</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Right Panel - Tool Details -->
|
||||
<div class="tool-details">
|
||||
<!-- Click2Crawl Details -->
|
||||
<div class="tool-content active" id="click2crawl">
|
||||
<div class="tool-header">
|
||||
<h3>🎯 Click2Crawl</h3>
|
||||
<span class="tool-tagline">Click elements to build extraction schemas - No LLM needed!</span>
|
||||
</div>
|
||||
|
||||
<div class="tool-steps">
|
||||
<div class="step-item">
|
||||
<div class="step-number">1</div>
|
||||
<div class="step-content">
|
||||
<h4>Select Container</h4>
|
||||
<p>Click on any repeating element like product cards or articles. Use up/down navigation to fine-tune selection!</p>
|
||||
<div class="step-visual">
|
||||
<span class="highlight-green">■</span> Container highlighted in green
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="step-item">
|
||||
<div class="step-number">2</div>
|
||||
<div class="step-content">
|
||||
<h4>Click Fields to Extract</h4>
|
||||
<p>Click on data fields inside the container - choose text, links, images, or attributes</p>
|
||||
<div class="step-visual">
|
||||
<span class="highlight-pink">■</span> Fields highlighted in pink
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="step-item">
|
||||
<div class="step-number">3</div>
|
||||
<div class="step-content">
|
||||
<h4>Test & Extract Data Instantly!</h4>
|
||||
<p>🎉 Click "Test Schema" to see extracted JSON immediately - no LLM or coding required!</p>
|
||||
<div class="step-visual">
|
||||
<span class="highlight-accent">⚡</span> See extracted JSON immediately
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="tool-features">
|
||||
<div class="feature-tag">🚀 Zero LLM dependency</div>
|
||||
<div class="feature-tag">📊 Instant JSON extraction</div>
|
||||
<div class="feature-tag">🎯 Visual element selection</div>
|
||||
<div class="feature-tag">🐍 Export Python code</div>
|
||||
<div class="feature-tag">✨ Live preview</div>
|
||||
<div class="feature-tag">📥 Download results</div>
|
||||
<div class="feature-tag">📝 Export to markdown</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Script Builder Details -->
|
||||
<div class="tool-content" id="script-builder">
|
||||
<div class="tool-header">
|
||||
<h3>🔴 Script Builder</h3>
|
||||
<span class="tool-tagline">Record actions, generate automation</span>
|
||||
</div>
|
||||
|
||||
<div class="tool-steps">
|
||||
<div class="step-item">
|
||||
<div class="step-number">1</div>
|
||||
<div class="step-content">
|
||||
<h4>Hit Record</h4>
|
||||
<p>Start capturing your browser interactions</p>
|
||||
<div class="step-visual">
|
||||
<span class="recording-dot">●</span> Recording indicator
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="step-item">
|
||||
<div class="step-number">2</div>
|
||||
<div class="step-content">
|
||||
<h4>Interact Naturally</h4>
|
||||
<p>Click, type, scroll - everything is captured</p>
|
||||
<div class="step-visual">
|
||||
<span class="action-icon">🖱️</span> <span class="action-icon">⌨️</span> <span class="action-icon">📜</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="step-item">
|
||||
<div class="step-number">3</div>
|
||||
<div class="step-content">
|
||||
<h4>Export Script</h4>
|
||||
<p>Get JavaScript for Crawl4AI's js_code parameter</p>
|
||||
<div class="step-visual">
|
||||
<span class="highlight-accent">📝</span> Automation ready
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="tool-features">
|
||||
<div class="feature-tag">Smart action grouping</div>
|
||||
<div class="feature-tag">Wait detection</div>
|
||||
<div class="feature-tag">Keyboard shortcuts</div>
|
||||
<div class="feature-tag alpha-tag">Alpha version</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Markdown Extraction Details -->
|
||||
<div class="tool-content" id="markdown-extraction">
|
||||
<div class="tool-header">
|
||||
<h3>📝 Markdown Extraction</h3>
|
||||
<span class="tool-tagline">Convert webpage content to clean markdown "as you see"</span>
|
||||
</div>
|
||||
|
||||
<div class="tool-steps">
|
||||
<div class="step-item">
|
||||
<div class="step-number">1</div>
|
||||
<div class="step-content">
|
||||
<h4>Ctrl/Cmd + Click</h4>
|
||||
<p>Hold Ctrl/Cmd and click multiple elements you want to extract</p>
|
||||
<div class="step-visual">
|
||||
<span class="highlight-green">🔢</span> Numbered selection badges
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="step-item">
|
||||
<div class="step-number">2</div>
|
||||
<div class="step-content">
|
||||
<h4>Enable Visual Text Mode</h4>
|
||||
<p>Extract content "as you see" - clean text without complex HTML structures</p>
|
||||
<div class="step-visual">
|
||||
<span class="highlight-accent">👁️</span> Visual Text Mode (As You See)
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="step-item">
|
||||
<div class="step-number">3</div>
|
||||
<div class="step-content">
|
||||
<h4>Export Clean Markdown</h4>
|
||||
<p>Get beautifully formatted markdown ready for documentation or LLMs</p>
|
||||
<div class="step-visual">
|
||||
<span class="highlight-pink">📄</span> Clean, readable output
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="tool-features">
|
||||
<div class="feature-tag">Multi-select with Ctrl/Cmd</div>
|
||||
<div class="feature-tag">Visual Text Mode (As You See)</div>
|
||||
<div class="feature-tag">Clean markdown output</div>
|
||||
<div class="feature-tag">Export to Crawl4AI Cloud (soon)</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Interactive Code Examples -->
|
||||
<section class="code-showcase">
|
||||
<h2>See the Generated Code & Extracted Data</h2>
|
||||
|
||||
<div class="code-tabs">
|
||||
<button class="code-tab active" data-example="schema">🎯 Click2Crawl</button>
|
||||
<button class="code-tab" data-example="script">🔴 Script Builder</button>
|
||||
<button class="code-tab" data-example="markdown">📝 Markdown Extraction</button>
|
||||
</div>
|
||||
|
||||
<div class="code-examples">
|
||||
<!-- Click2Crawl Code -->
|
||||
<div class="code-example active" id="code-schema">
|
||||
<div style="display: grid; grid-template-columns: 1fr 1fr; gap: 16px;">
|
||||
<!-- Python Code -->
|
||||
<div class="terminal-window">
|
||||
<div class="terminal-header">
|
||||
<span class="terminal-title">click2crawl_extraction.py</span>
|
||||
<button class="copy-button" data-code="schema-python">Copy</button>
|
||||
</div>
|
||||
<div class="terminal-content">
|
||||
<pre><code><span class="comment">#!/usr/bin/env python3</span>
|
||||
<span class="comment">"""
|
||||
🎉 NO LLM NEEDED! Direct extraction with CSS selectors
|
||||
Generated by Crawl4AI Chrome Extension - Click2Crawl
|
||||
"""</span>
|
||||
|
||||
<span class="keyword">import</span> asyncio
|
||||
<span class="keyword">import</span> json
|
||||
<span class="keyword">from</span> crawl4ai <span class="keyword">import</span> AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
<span class="keyword">from</span> crawl4ai.extraction_strategy <span class="keyword">import</span> JsonCssExtractionStrategy
|
||||
|
||||
<span class="comment"># The EXACT schema from Click2Crawl - no guessing!</span>
|
||||
EXTRACTION_SCHEMA = {
|
||||
<span class="string">"name"</span>: <span class="string">"Product Catalog"</span>,
|
||||
<span class="string">"baseSelector"</span>: <span class="string">"div.product-card"</span>, <span class="comment"># The container you selected</span>
|
||||
<span class="string">"fields"</span>: [
|
||||
{
|
||||
<span class="string">"name"</span>: <span class="string">"title"</span>,
|
||||
<span class="string">"selector"</span>: <span class="string">"h3.product-title"</span>,
|
||||
<span class="string">"type"</span>: <span class="string">"text"</span>
|
||||
},
|
||||
{
|
||||
<span class="string">"name"</span>: <span class="string">"price"</span>,
|
||||
<span class="string">"selector"</span>: <span class="string">"span.price"</span>,
|
||||
<span class="string">"type"</span>: <span class="string">"text"</span>
|
||||
},
|
||||
{
|
||||
<span class="string">"name"</span>: <span class="string">"image"</span>,
|
||||
<span class="string">"selector"</span>: <span class="string">"img.product-img"</span>,
|
||||
<span class="string">"type"</span>: <span class="string">"attribute"</span>,
|
||||
<span class="string">"attribute"</span>: <span class="string">"src"</span>
|
||||
},
|
||||
{
|
||||
<span class="string">"name"</span>: <span class="string">"link"</span>,
|
||||
<span class="string">"selector"</span>: <span class="string">"a.product-link"</span>,
|
||||
<span class="string">"type"</span>: <span class="string">"attribute"</span>,
|
||||
<span class="string">"attribute"</span>: <span class="string">"href"</span>
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
<span class="keyword">async</span> <span class="keyword">def</span> <span class="function">extract_data</span>(url: str):
|
||||
<span class="comment"># Direct extraction - no LLM API calls!</span>
|
||||
extraction_strategy = JsonCssExtractionStrategy(schema=EXTRACTION_SCHEMA)
|
||||
|
||||
<span class="keyword">async</span> <span class="keyword">with</span> AsyncWebCrawler() <span class="keyword">as</span> crawler:
|
||||
result = <span class="keyword">await</span> crawler.arun(
|
||||
url=url,
|
||||
config=CrawlerRunConfig(extraction_strategy=extraction_strategy)
|
||||
)
|
||||
|
||||
<span class="keyword">if</span> result.success:
|
||||
data = json.loads(result.extracted_content)
|
||||
<span class="keyword">print</span>(<span class="string">f"✅ Extracted {len(data)} items instantly!"</span>)
|
||||
|
||||
<span class="comment"># Save to file</span>
|
||||
<span class="keyword">with</span> open(<span class="string">'products.json'</span>, <span class="string">'w'</span>) <span class="keyword">as</span> f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
<span class="keyword">return</span> data
|
||||
|
||||
<span class="comment"># Run extraction on any similar page!</span>
|
||||
data = asyncio.run(extract_data(<span class="string">"https://example.com/products"</span>))
|
||||
|
||||
<span class="comment"># 🎯 Result: Clean JSON data, no LLM costs, instant results!</span></code></pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Extracted JSON Data -->
|
||||
<div class="terminal-window">
|
||||
<div class="terminal-header">
|
||||
<span class="terminal-title">extracted_data.json</span>
|
||||
<button class="copy-button" data-code="schema-json">Copy</button>
|
||||
</div>
|
||||
<div class="terminal-content">
|
||||
<pre><code><span class="comment">// 🎉 Instantly extracted from the page - no coding required!</span>
|
||||
[
|
||||
{
|
||||
<span class="string">"title"</span>: <span class="string">"Wireless Bluetooth Headphones"</span>,
|
||||
<span class="string">"price"</span>: <span class="string">"$79.99"</span>,
|
||||
<span class="string">"image"</span>: <span class="string">"https://example.com/images/headphones-bt-01.jpg"</span>,
|
||||
<span class="string">"link"</span>: <span class="string">"/products/wireless-bluetooth-headphones"</span>
|
||||
},
|
||||
{
|
||||
<span class="string">"title"</span>: <span class="string">"Smart Watch Pro 2024"</span>,
|
||||
<span class="string">"price"</span>: <span class="string">"$299.00"</span>,
|
||||
<span class="string">"image"</span>: <span class="string">"https://example.com/images/smartwatch-pro.jpg"</span>,
|
||||
<span class="string">"link"</span>: <span class="string">"/products/smart-watch-pro-2024"</span>
|
||||
},
|
||||
{
|
||||
<span class="string">"title"</span>: <span class="string">"4K Webcam for Streaming"</span>,
|
||||
<span class="string">"price"</span>: <span class="string">"$149.99"</span>,
|
||||
<span class="string">"image"</span>: <span class="string">"https://example.com/images/webcam-4k.jpg"</span>,
|
||||
<span class="string">"link"</span>: <span class="string">"/products/4k-webcam-streaming"</span>
|
||||
},
|
||||
{
|
||||
<span class="string">"title"</span>: <span class="string">"Mechanical Gaming Keyboard RGB"</span>,
|
||||
<span class="string">"price"</span>: <span class="string">"$129.99"</span>,
|
||||
<span class="string">"image"</span>: <span class="string">"https://example.com/images/keyboard-gaming.jpg"</span>,
|
||||
<span class="string">"link"</span>: <span class="string">"/products/mechanical-gaming-keyboard"</span>
|
||||
},
|
||||
{
|
||||
<span class="string">"title"</span>: <span class="string">"USB-C Hub 7-in-1"</span>,
|
||||
<span class="string">"price"</span>: <span class="string">"$45.99"</span>,
|
||||
<span class="string">"image"</span>: <span class="string">"https://example.com/images/usbc-hub.jpg"</span>,
|
||||
<span class="string">"link"</span>: <span class="string">"/products/usb-c-hub-7in1"</span>
|
||||
}
|
||||
]</code></pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Script Builder Code -->
|
||||
<div class="code-example" id="code-script">
|
||||
<div class="terminal-window">
|
||||
<div class="terminal-header">
|
||||
<span class="terminal-title">automation_script.py</span>
|
||||
<button class="copy-button" data-code="script">Copy</button>
|
||||
</div>
|
||||
<div class="terminal-content">
|
||||
<pre><code><span class="keyword">import</span> asyncio
|
||||
<span class="keyword">from</span> crawl4ai <span class="keyword">import</span> AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
<span class="comment"># JavaScript generated from your recorded actions</span>
|
||||
js_script = <span class="string">"""
|
||||
// Search for products
|
||||
document.querySelector('button.search-toggle').click();
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
|
||||
// Type search query
|
||||
const searchInput = document.querySelector('input#search');
|
||||
searchInput.value = 'wireless headphones';
|
||||
searchInput.dispatchEvent(new Event('input', {bubbles: true}));
|
||||
|
||||
// Submit search
|
||||
searchInput.dispatchEvent(new KeyboardEvent('keydown', {
|
||||
key: 'Enter', keyCode: 13, bubbles: true
|
||||
}));
|
||||
|
||||
// Wait for results
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
|
||||
// Click first product
|
||||
document.querySelector('.product-item:first-child').click();
|
||||
|
||||
// Wait for product page
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Add to cart
|
||||
document.querySelector('button.add-to-cart').click();
|
||||
"""</span>
|
||||
|
||||
<span class="keyword">async</span> <span class="keyword">def</span> <span class="function">automate_shopping</span>():
|
||||
config = CrawlerRunConfig(
|
||||
js_code=js_script,
|
||||
wait_for=<span class="string">"css:.cart-confirmation"</span>,
|
||||
screenshot=<span class="keyword">True</span>
|
||||
)
|
||||
|
||||
<span class="keyword">async</span> <span class="keyword">with</span> AsyncWebCrawler() <span class="keyword">as</span> crawler:
|
||||
result = <span class="keyword">await</span> crawler.arun(
|
||||
url=<span class="string">"https://shop.example.com"</span>,
|
||||
config=config
|
||||
)
|
||||
<span class="keyword">print</span>(<span class="string">f"✓ Automation complete: {result.url}"</span>)
|
||||
<span class="keyword">return</span> result
|
||||
|
||||
asyncio.run(automate_shopping())</code></pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Markdown Extraction Output -->
|
||||
<div class="code-example" id="code-markdown">
|
||||
<div class="terminal-window">
|
||||
<div class="terminal-header">
|
||||
<span class="terminal-title">extracted_content.md</span>
|
||||
<button class="copy-button" data-code="markdown">Copy</button>
|
||||
</div>
|
||||
<div class="terminal-content">
|
||||
<pre><code><span class="comment"># Extracted from Hacker News with Visual Text Mode 👁️</span>
|
||||
|
||||
<span class="string">1. **Show HN: I built a tool to find and reach out to YouTubers** (hellosimply.io)
|
||||
84 points by erickim 2 hours ago | hide | 31 comments
|
||||
|
||||
2. **The 24 Hour Restaurant** (logicmag.io)
|
||||
124 points by helsinkiandrew 5 hours ago | hide | 52 comments
|
||||
|
||||
3. **Building a Better Bloom Filter in Rust** (carlmastrangelo.com)
|
||||
89 points by carlmastrangelo 3 hours ago | hide | 27 comments
|
||||
|
||||
---
|
||||
|
||||
### Article: The 24 Hour Restaurant
|
||||
|
||||
In New York City, the 24-hour restaurant is becoming extinct. What we lose when we can no longer eat whenever we want.
|
||||
|
||||
When I first moved to New York, I loved that I could get a full meal at 3 AM. Not just pizza or fast food, but a proper sit-down dinner with table service and a menu that ran for pages. The city that never sleeps had restaurants that matched its rhythm.
|
||||
|
||||
Today, finding a 24-hour restaurant in Manhattan requires genuine effort. The pandemic accelerated a decline that was already underway, but the roots go deeper: rising rents, changing labor laws, and shifting cultural patterns have all contributed to the death of round-the-clock dining.
|
||||
|
||||
---
|
||||
|
||||
### Product Review: Framework Laptop 16
|
||||
|
||||
**Specifications:**
|
||||
- Display: 16" 2560×1600 165Hz
|
||||
- Processor: AMD Ryzen 7 7840HS
|
||||
- Memory: 32GB DDR5-5600
|
||||
- Storage: 2TB NVMe Gen4
|
||||
- Price: Starting at $1,399
|
||||
|
||||
**Pros:**
|
||||
- Fully modular and repairable
|
||||
- Excellent Linux support
|
||||
- Great keyboard and trackpad
|
||||
- Expansion card system
|
||||
|
||||
**Cons:**
|
||||
- Battery life could be better
|
||||
- Slightly heavier than competitors
|
||||
- Fan noise under load</span></code></pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
<!-- Crawl4AI Cloud Section -->
|
||||
<section class="cloud-section">
|
||||
<div class="cloud-announcement">
|
||||
<h2>Crawl4AI Cloud</h2>
|
||||
<p class="cloud-tagline">Your browser cluster without the cluster.</p>
|
||||
|
||||
<div class="cloud-features-preview">
|
||||
<div class="cloud-feature-item">
|
||||
⚡ POST /crawl
|
||||
</div>
|
||||
<div class="cloud-feature-item">
|
||||
🌐 JS-rendered pages
|
||||
</div>
|
||||
<div class="cloud-feature-item">
|
||||
📊 Schema extraction built-in
|
||||
</div>
|
||||
<div class="cloud-feature-item">
|
||||
💰 $0.001/page
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button class="cloud-cta-button" id="joinWaitlist">
|
||||
Get Early Access →
|
||||
</button>
|
||||
|
||||
<p class="cloud-hint">See it extract your own data. Right now.</p>
|
||||
</div>
|
||||
|
||||
<!-- Hidden Signup Form -->
|
||||
<div class="signup-overlay" id="signupOverlay">
|
||||
<div class="signup-container" id="signupContainer">
|
||||
<button class="close-signup" id="closeSignup">×</button>
|
||||
|
||||
<div class="signup-content" id="signupForm">
|
||||
<h3>🚀 Join C4AI Cloud Waiting List</h3>
|
||||
<p>Be among the first to experience the future of web scraping</p>
|
||||
|
||||
<form id="waitlistForm" class="waitlist-form">
|
||||
<div class="form-field">
|
||||
<label for="userName">Your Name</label>
|
||||
<input type="text" id="userName" name="name" placeholder="John Doe" required>
|
||||
</div>
|
||||
|
||||
<div class="form-field">
|
||||
<label for="userEmail">Email Address</label>
|
||||
<input type="email" id="userEmail" name="email" placeholder="john@example.com" required>
|
||||
</div>
|
||||
|
||||
<div class="form-field">
|
||||
<label for="userCompany">Company (Optional)</label>
|
||||
<input type="text" id="userCompany" name="company" placeholder="Acme Inc.">
|
||||
</div>
|
||||
|
||||
<div class="form-field">
|
||||
<label for="useCase">What will you use Crawl4AI Cloud for?</label>
|
||||
<select id="useCase" name="useCase">
|
||||
<option value="">Select use case...</option>
|
||||
<option value="price-monitoring">Price Monitoring</option>
|
||||
<option value="news-aggregation">News Aggregation</option>
|
||||
<option value="market-research">Market Research</option>
|
||||
<option value="ai-training">AI Training Data</option>
|
||||
<option value="other">Other</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<button type="submit" class="submit-button">
|
||||
<span>🎯</span> Submit & Watch the Magic
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<!-- Crawling Animation -->
|
||||
<div class="crawl-animation" id="crawlAnimation" style="display: none;">
|
||||
<div class="terminal-window crawl-terminal">
|
||||
<div class="terminal-header">
|
||||
<span class="terminal-title">Crawl4AI Cloud Demo</span>
|
||||
</div>
|
||||
<div class="terminal-content">
|
||||
<pre id="crawlOutput" class="crawl-log"><code>$ crawl4ai cloud extract --url "signup-form" --auto-detect</code></pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="extracted-preview" id="extractedPreview" style="display: none;">
|
||||
<h4>📊 Extracted Data</h4>
|
||||
<pre class="json-preview"><code id="jsonOutput"></code></pre>
|
||||
</div>
|
||||
|
||||
<div class="success-message" id="successMessage" style="display: none;">
|
||||
<div class="success-icon">✅</div>
|
||||
<h3>Data Uploaded Successfully!</h3>
|
||||
<p>You're on the Crawl4AI Cloud waiting list!</p>
|
||||
<p>What you just witnessed:</p>
|
||||
<ul>
|
||||
<li>⚡ Real-time extraction of your form data</li>
|
||||
<li>🔄 Automatic schema detection</li>
|
||||
<li>📤 Instant cloud processing</li>
|
||||
<li>✨ No code required - just like that!</li>
|
||||
</ul>
|
||||
<p class="success-note">We'll notify you at <strong id="userEmailDisplay"></strong> when Crawl4AI Cloud launches!</p>
|
||||
<button class="continue-button" id="continueBtn">Continue Exploring</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Coming Soon Section -->
|
||||
<section class="coming-soon-section">
|
||||
<h2>More Features Coming Soon</h2>
|
||||
<div class="terminal-window">
|
||||
<div class="terminal-header">
|
||||
<span class="terminal-title">Roadmap</span>
|
||||
</div>
|
||||
<div class="terminal-content">
|
||||
<p class="intro-text">We're continuously expanding C4AI Assistant with powerful new features:</p>
|
||||
|
||||
<div class="coming-features">
|
||||
<div class="coming-feature">
|
||||
<div class="feature-header">
|
||||
<span class="feature-badge">Direct</span>
|
||||
<h3>Direct Data Download</h3>
|
||||
</div>
|
||||
<p>Skip the code generation entirely! Download extracted data directly from Click2Crawl as JSON or CSV files.</p>
|
||||
<div class="feature-preview">
|
||||
<code>📊 One-click download • No Python needed • Multiple export formats</code>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="coming-feature">
|
||||
<div class="feature-header">
|
||||
<span class="feature-badge">AI</span>
|
||||
<h3>Smart Field Detection</h3>
|
||||
</div>
|
||||
<p>AI-powered field detection for Click2Crawl that automatically suggests the most likely data fields on any page.</p>
|
||||
<div class="feature-preview">
|
||||
<code>🤖 Auto-detect fields • Smart naming • Pattern recognition</code>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="stay-tuned">
|
||||
<p>🚀 Stay tuned for updates! Follow our <a href="https://github.com/unclecode/crawl4ai" target="_blank">GitHub</a> for the latest releases.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Footer -->
|
||||
<footer class="footer">
|
||||
<div class="footer-content">
|
||||
<div class="footer-section">
|
||||
<h4>Resources</h4>
|
||||
<ul>
|
||||
<li><a href="https://github.com/unclecode/crawl4ai">GitHub Repository</a></li>
|
||||
<li><a href="../../">Documentation</a></li>
|
||||
<li><a href="https://discord.gg/jP8KfhDhyN">Discord Community</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="footer-section">
|
||||
<h4>Connect</h4>
|
||||
<ul>
|
||||
<li><a href="https://twitter.com/unclecode">Twitter @unclecode</a></li>
|
||||
<li><a href="https://github.com/unclecode">GitHub @unclecode</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<div class="footer-bottom">
|
||||
<p>Made with 🚀 by the Crawl4AI team</p>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Tool Selector Interaction
|
||||
document.querySelectorAll('.tool-selector').forEach(selector => {
|
||||
selector.addEventListener('click', function() {
|
||||
// Remove active class from all selectors
|
||||
document.querySelectorAll('.tool-selector').forEach(s => s.classList.remove('active'));
|
||||
document.querySelectorAll('.tool-content').forEach(c => c.classList.remove('active'));
|
||||
|
||||
// Add active class to clicked selector
|
||||
this.classList.add('active');
|
||||
|
||||
// Show corresponding content
|
||||
const toolId = this.getAttribute('data-tool');
|
||||
const contentElement = document.getElementById(toolId);
|
||||
if (contentElement) {
|
||||
contentElement.classList.add('active');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Code Tab Interaction
|
||||
document.querySelectorAll('.code-tab').forEach(tab => {
|
||||
tab.addEventListener('click', function() {
|
||||
// Remove active class from all tabs
|
||||
document.querySelectorAll('.code-tab').forEach(t => t.classList.remove('active'));
|
||||
document.querySelectorAll('.code-example').forEach(e => e.classList.remove('active'));
|
||||
|
||||
// Add active class to clicked tab
|
||||
this.classList.add('active');
|
||||
|
||||
// Show corresponding code
|
||||
const exampleId = this.getAttribute('data-example');
|
||||
document.getElementById('code-' + exampleId).classList.add('active');
|
||||
});
|
||||
});
|
||||
|
||||
// Copy Button Functionality
|
||||
document.querySelectorAll('.copy-button').forEach(button => {
|
||||
button.addEventListener('click', async function() {
|
||||
const codeType = this.getAttribute('data-code');
|
||||
let codeText = '';
|
||||
|
||||
// Handle different code types
|
||||
if (codeType === 'schema-python') {
|
||||
const codeElement = document.querySelector('#code-schema .terminal-window:first-child pre code');
|
||||
codeText = codeElement.textContent;
|
||||
} else if (codeType === 'schema-json') {
|
||||
const codeElement = document.querySelector('#code-schema .terminal-window:last-child pre code');
|
||||
codeText = codeElement.textContent;
|
||||
} else {
|
||||
const codeElement = document.getElementById('code-' + codeType).querySelector('pre code');
|
||||
codeText = codeElement.textContent;
|
||||
}
|
||||
|
||||
try {
|
||||
await navigator.clipboard.writeText(codeText);
|
||||
this.textContent = 'Copied!';
|
||||
this.classList.add('copied');
|
||||
|
||||
setTimeout(() => {
|
||||
this.textContent = 'Copy';
|
||||
this.classList.remove('copied');
|
||||
}, 2000);
|
||||
} catch (err) {
|
||||
console.error('Failed to copy code:', err);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Crawl4AI Cloud Interactive Demo
|
||||
const joinWaitlistBtn = document.getElementById('joinWaitlist');
|
||||
const signupOverlay = document.getElementById('signupOverlay');
|
||||
const closeSignupBtn = document.getElementById('closeSignup');
|
||||
const waitlistForm = document.getElementById('waitlistForm');
|
||||
const signupForm = document.getElementById('signupForm');
|
||||
const crawlAnimation = document.getElementById('crawlAnimation');
|
||||
const crawlOutput = document.getElementById('crawlOutput');
|
||||
const extractedPreview = document.getElementById('extractedPreview');
|
||||
const jsonOutput = document.getElementById('jsonOutput');
|
||||
const successMessage = document.getElementById('successMessage');
|
||||
const continueBtn = document.getElementById('continueBtn');
|
||||
const userEmailDisplay = document.getElementById('userEmailDisplay');
|
||||
|
||||
// Open signup modal
|
||||
joinWaitlistBtn.addEventListener('click', () => {
|
||||
signupOverlay.classList.add('active');
|
||||
});
|
||||
|
||||
// Banner button
|
||||
const joinWaitlistBannerBtn = document.getElementById('joinWaitlistBanner');
|
||||
if (joinWaitlistBannerBtn) {
|
||||
joinWaitlistBannerBtn.addEventListener('click', () => {
|
||||
signupOverlay.classList.add('active');
|
||||
});
|
||||
}
|
||||
|
||||
// Close signup modal
|
||||
closeSignupBtn.addEventListener('click', () => {
|
||||
signupOverlay.classList.remove('active');
|
||||
});
|
||||
|
||||
// Close on overlay click
|
||||
signupOverlay.addEventListener('click', (e) => {
|
||||
if (e.target === signupOverlay) {
|
||||
signupOverlay.classList.remove('active');
|
||||
}
|
||||
});
|
||||
|
||||
// Continue button
|
||||
if (continueBtn) {
|
||||
continueBtn.addEventListener('click', () => {
|
||||
signupOverlay.classList.remove('active');
|
||||
// Reset form for next time
|
||||
waitlistForm.reset();
|
||||
signupForm.style.display = 'block';
|
||||
crawlAnimation.style.display = 'none';
|
||||
extractedPreview.style.display = 'none';
|
||||
successMessage.style.display = 'none';
|
||||
});
|
||||
}
|
||||
|
||||
// Form submission with crawling animation
|
||||
waitlistForm.addEventListener('submit', async (e) => {
|
||||
e.preventDefault();
|
||||
|
||||
// Get form data
|
||||
const formData = {
|
||||
name: document.getElementById('userName').value,
|
||||
email: document.getElementById('userEmail').value,
|
||||
company: document.getElementById('userCompany').value || 'Not specified',
|
||||
useCase: document.getElementById('useCase').value || 'General web scraping',
|
||||
timestamp: new Date().toISOString(),
|
||||
source: 'Crawl4AI Assistant Landing Page'
|
||||
};
|
||||
|
||||
// Update email display
|
||||
userEmailDisplay.textContent = formData.email;
|
||||
|
||||
// Hide form and show crawling animation
|
||||
signupForm.style.display = 'none';
|
||||
crawlAnimation.style.display = 'block';
|
||||
|
||||
// Clear previous output
|
||||
const codeElement = crawlOutput.querySelector('code');
|
||||
codeElement.innerHTML = '$ crawl4ai cloud extract --url "signup-form" --auto-detect\n\n';
|
||||
|
||||
// Simulate crawling process with proper C4AI log format
|
||||
const crawlSteps = [
|
||||
{
|
||||
log: '<span class="log-init">[INIT]....</span> → Crawl4AI Cloud 1.0.0',
|
||||
time: '0.12s'
|
||||
},
|
||||
{
|
||||
log: '<span class="log-fetch">[FETCH]...</span> ↓ https://crawl4ai.com/waitlist-form',
|
||||
time: '0.45s'
|
||||
},
|
||||
{
|
||||
log: '<span class="log-scrape">[SCRAPE]..</span> ◆ https://crawl4ai.com/waitlist-form',
|
||||
time: '0.28s'
|
||||
},
|
||||
{
|
||||
log: '<span class="log-extract">[EXTRACT].</span> ■ Extracting form data with auto-detect',
|
||||
time: '0.55s'
|
||||
},
|
||||
{
|
||||
log: '<span class="log-complete">[COMPLETE]</span> ● https://crawl4ai.com/waitlist-form',
|
||||
time: '1.40s'
|
||||
}
|
||||
];
|
||||
|
||||
let stepIndex = 0;
|
||||
const typeStep = async () => {
|
||||
if (stepIndex < crawlSteps.length) {
|
||||
const step = crawlSteps[stepIndex];
|
||||
codeElement.innerHTML += step.log + ' | <span class="log-success">✓</span> | <span class="log-time">⏱: ' + step.time + '</span>\n';
|
||||
stepIndex++;
|
||||
|
||||
// Scroll to bottom
|
||||
const terminal = crawlOutput.parentElement;
|
||||
terminal.scrollTop = terminal.scrollHeight;
|
||||
|
||||
setTimeout(typeStep, 600);
|
||||
} else {
|
||||
// Show extracted data
|
||||
setTimeout(() => {
|
||||
codeElement.innerHTML += '\n<span class="log-success">[UPLOAD]..</span> ↑ Uploading to Crawl4AI Cloud...';
|
||||
|
||||
setTimeout(() => {
|
||||
extractedPreview.style.display = 'block';
|
||||
jsonOutput.textContent = JSON.stringify(formData, null, 2);
|
||||
|
||||
// Add syntax highlighting
|
||||
jsonOutput.innerHTML = jsonOutput.textContent
|
||||
.replace(/"([^"]+)":/g, '<span class="string">"$1"</span>:')
|
||||
.replace(/: "([^"]+)"/g, ': <span class="string">"$1"</span>');
|
||||
|
||||
codeElement.innerHTML += ' | <span class="log-success">✓</span> | <span class="log-time">⏱: 0.23s</span>\n';
|
||||
codeElement.innerHTML += '\n<span class="log-success">[SUCCESS]</span> ✨ Data uploaded successfully!';
|
||||
|
||||
// Show success message after a delay
|
||||
setTimeout(() => {
|
||||
successMessage.style.display = 'block';
|
||||
|
||||
// Smooth scroll to bottom to show success message
|
||||
setTimeout(() => {
|
||||
const container = document.getElementById('signupContainer');
|
||||
container.scrollTo({
|
||||
top: container.scrollHeight,
|
||||
behavior: 'smooth'
|
||||
});
|
||||
}, 100);
|
||||
|
||||
// Actually submit to waiting list (you can implement this)
|
||||
console.log('Waitlist submission:', formData);
|
||||
}, 1500);
|
||||
}, 800);
|
||||
}, 600);
|
||||
}
|
||||
};
|
||||
|
||||
// Start the animation
|
||||
setTimeout(typeStep, 500);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "Crawl4AI Assistant",
|
||||
"version": "1.3.0",
|
||||
"description": "Visual schema and script builder for Crawl4AI - Build extraction schemas and automation scripts by clicking and recording actions",
|
||||
"permissions": [
|
||||
"activeTab",
|
||||
"storage",
|
||||
"downloads"
|
||||
],
|
||||
"host_permissions": [
|
||||
"<all_urls>"
|
||||
],
|
||||
"action": {
|
||||
"default_popup": "popup/popup.html",
|
||||
"default_icon": {
|
||||
"16": "icons/icon-16.png",
|
||||
"48": "icons/icon-48.png",
|
||||
"128": "icons/icon-128.png"
|
||||
}
|
||||
},
|
||||
"content_scripts": [
|
||||
{
|
||||
"matches": ["<all_urls>"],
|
||||
"js": [
|
||||
"libs/marked.min.js",
|
||||
"content/shared/utils.js",
|
||||
"content/markdownPreviewModal.js",
|
||||
"content/click2crawl.js",
|
||||
"content/scriptBuilder.js",
|
||||
"content/contentAnalyzer.js",
|
||||
"content/markdownConverter.js",
|
||||
"content/markdownExtraction.js",
|
||||
"content/content.js"
|
||||
],
|
||||
"css": ["content/overlay.css"],
|
||||
"run_at": "document_idle"
|
||||
}
|
||||
],
|
||||
"background": {
|
||||
"service_worker": "background/service-worker.js"
|
||||
},
|
||||
"icons": {
|
||||
"16": "icons/icon-16.png",
|
||||
"48": "icons/icon-48.png",
|
||||
"128": "icons/icon-128.png"
|
||||
},
|
||||
"web_accessible_resources": [
|
||||
{
|
||||
"resources": ["icons/*", "assets/*"],
|
||||
"matches": ["<all_urls>"]
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 3.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
@@ -0,0 +1,330 @@
|
||||
/* Font Face Definitions */
|
||||
@font-face {
|
||||
font-family: 'Dank Mono';
|
||||
src: url('../assets/DankMono-Regular.woff2') format('woff2');
|
||||
font-weight: 400;
|
||||
font-style: normal;
|
||||
font-display: swap;
|
||||
}
|
||||
|
||||
@font-face {
|
||||
font-family: 'Dank Mono';
|
||||
src: url('../assets/DankMono-Bold.woff2') format('woff2');
|
||||
font-weight: 700;
|
||||
font-style: normal;
|
||||
font-display: swap;
|
||||
}
|
||||
|
||||
@font-face {
|
||||
font-family: 'Dank Mono';
|
||||
src: url('../assets/DankMono-Italic.woff2') format('woff2');
|
||||
font-weight: 400;
|
||||
font-style: italic;
|
||||
font-display: swap;
|
||||
}
|
||||
|
||||
:root {
|
||||
--font-primary: 'Dank Mono', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, monospace;
|
||||
}
|
||||
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
width: 380px;
|
||||
font-family: var(--font-primary);
|
||||
background: #0a0a0a;
|
||||
color: #e0e0e0;
|
||||
border-radius: 16px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.popup-container {
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
margin-bottom: 20px;
|
||||
padding-bottom: 16px;
|
||||
border-bottom: 1px solid #2a2a2a;
|
||||
}
|
||||
|
||||
.logo {
|
||||
width: 48px;
|
||||
height: 48px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.header-content {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
header h1 {
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
color: #fff;
|
||||
margin: 0 0 4px 0;
|
||||
}
|
||||
|
||||
.header-stats {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.github-stars {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
color: #999;
|
||||
text-decoration: none;
|
||||
font-size: 13px;
|
||||
transition: color 0.2s ease;
|
||||
}
|
||||
|
||||
.github-stars:hover {
|
||||
color: #4a9eff;
|
||||
}
|
||||
|
||||
.github-icon {
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.mode-selector {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.mode-button {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
padding: 16px;
|
||||
background: #1a1a1a;
|
||||
border: 2px solid #2a2a2a;
|
||||
border-radius: 12px;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
width: 100%;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.mode-button:hover:not(:disabled) {
|
||||
background: #252525;
|
||||
border-color: #4a9eff;
|
||||
transform: translateY(-2px);
|
||||
}
|
||||
|
||||
.mode-button:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.mode-button .icon {
|
||||
font-size: 32px;
|
||||
width: 48px;
|
||||
height: 48px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
background: #252525;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.mode-button.schema .icon {
|
||||
background: #1e3a5f;
|
||||
}
|
||||
|
||||
.mode-button.script .icon {
|
||||
background: #3a1e5f;
|
||||
}
|
||||
|
||||
.mode-button.c2c .icon {
|
||||
background: #1e5f3a;
|
||||
}
|
||||
|
||||
.mode-info h3 {
|
||||
font-size: 16px;
|
||||
color: #fff;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.mode-info p {
|
||||
font-size: 13px;
|
||||
color: #999;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.active-session {
|
||||
background: #1a1a1a;
|
||||
border: 2px solid #4a9eff;
|
||||
border-radius: 12px;
|
||||
padding: 16px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.active-session.hidden {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.session-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.status-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
background: #4a9eff;
|
||||
border-radius: 50%;
|
||||
animation: pulse 2s infinite;
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; }
|
||||
50% { opacity: 0.5; }
|
||||
}
|
||||
|
||||
.session-title {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: #4a9eff;
|
||||
}
|
||||
|
||||
.session-stats {
|
||||
display: flex;
|
||||
gap: 20px;
|
||||
margin-bottom: 16px;
|
||||
padding: 12px;
|
||||
background: #0a0a0a;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.stat {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.stat-label {
|
||||
display: block;
|
||||
font-size: 11px;
|
||||
color: #666;
|
||||
text-transform: uppercase;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.stat-value {
|
||||
font-size: 14px;
|
||||
color: #fff;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.session-actions {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.action-button {
|
||||
flex: 1;
|
||||
padding: 10px 16px;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.action-button.primary {
|
||||
background: #4a9eff;
|
||||
color: #000;
|
||||
}
|
||||
|
||||
.action-button.primary:hover:not(:disabled) {
|
||||
background: #3a8eef;
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.action-button.primary:disabled {
|
||||
background: #2a4a7f;
|
||||
color: #666;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.action-button.secondary {
|
||||
background: #2a2a2a;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.action-button.secondary:hover {
|
||||
background: #3a3a3a;
|
||||
}
|
||||
|
||||
.instructions {
|
||||
background: #1a1a1a;
|
||||
border-radius: 12px;
|
||||
padding: 16px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.instructions h4 {
|
||||
font-size: 14px;
|
||||
margin-bottom: 12px;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.instructions ol {
|
||||
padding-left: 20px;
|
||||
}
|
||||
|
||||
.instructions li {
|
||||
font-size: 13px;
|
||||
line-height: 1.6;
|
||||
color: #ccc;
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
|
||||
footer {
|
||||
padding-top: 16px;
|
||||
border-top: 1px solid #2a2a2a;
|
||||
}
|
||||
|
||||
.social-links {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.social-link {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
color: #999;
|
||||
text-decoration: none;
|
||||
font-size: 12px;
|
||||
transition: all 0.2s ease;
|
||||
padding: 6px 12px;
|
||||
border-radius: 6px;
|
||||
background: #1a1a1a;
|
||||
}
|
||||
|
||||
.social-link:hover {
|
||||
color: #0fbbaa;
|
||||
background: #2a2a2a;
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.social-link svg {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<link rel="stylesheet" href="popup.css">
|
||||
</head>
|
||||
<body>
|
||||
<div class="popup-container">
|
||||
<header>
|
||||
<img src="icons/icon-48.png" class="logo" alt="Crawl4AI">
|
||||
<div class="header-content">
|
||||
<h1>Crawl4AI Assistant</h1>
|
||||
<div class="header-stats">
|
||||
<a href="https://github.com/unclecode/crawl4ai" target="_blank" class="github-stars">
|
||||
<svg class="github-icon" viewBox="0 0 16 16" width="16" height="16">
|
||||
<path fill="currentColor" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
|
||||
</svg>
|
||||
<span id="stars-count">Loading...</span>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="mode-selector">
|
||||
<button id="schema-mode" class="mode-button schema">
|
||||
<div class="icon">🎯</div>
|
||||
<div class="mode-info">
|
||||
<h3>Click2Crawl</h3>
|
||||
<p>Click elements to build extraction schemas</p>
|
||||
</div>
|
||||
</button>
|
||||
|
||||
<button id="script-mode" class="mode-button script">
|
||||
<div class="icon">🔴</div>
|
||||
<div class="mode-info">
|
||||
<h3>Script Builder <span style="color: #ff3c74; font-size: 10px;">(Alpha)</span></h3>
|
||||
<p>Record actions to build automation scripts</p>
|
||||
</div>
|
||||
</button>
|
||||
|
||||
<button id="c2c-mode" class="mode-button c2c">
|
||||
<div class="icon">📝</div>
|
||||
<div class="mode-info">
|
||||
<h3>Markdown Extraction</h3>
|
||||
<p>Select elements and convert to clean markdown</p>
|
||||
</div>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div id="active-session" class="active-session hidden">
|
||||
<div class="session-header">
|
||||
<span class="status-dot"></span>
|
||||
<span class="session-title">Schema Capture Active</span>
|
||||
</div>
|
||||
<div class="session-stats">
|
||||
<div class="stat">
|
||||
<span class="stat-label">Container:</span>
|
||||
<span id="container-status" class="stat-value">Not selected</span>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<span class="stat-label">Fields:</span>
|
||||
<span id="fields-count" class="stat-value">0</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="session-actions">
|
||||
<button id="generate-code" class="action-button primary" disabled>
|
||||
<span>Generate Code</span>
|
||||
</button>
|
||||
<button id="stop-capture" class="action-button secondary">
|
||||
<span>Stop Capture</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="instructions" style="display: none;">
|
||||
<h4>How to use:</h4>
|
||||
<ol>
|
||||
<li>Click "Click2Crawl" to start</li>
|
||||
<li>Click on a container element (e.g., product card)</li>
|
||||
<li>Click individual fields inside and name them</li>
|
||||
<li>Generate Python code when done</li>
|
||||
</ol>
|
||||
</div>
|
||||
|
||||
<footer>
|
||||
<div class="social-links">
|
||||
<a href="https://docs.crawl4ai.com" target="_blank" class="social-link">
|
||||
<svg viewBox="0 0 24 24" width="16" height="16">
|
||||
<path fill="currentColor" d="M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2zm-2 15l-5-5 1.41-1.41L10 14.17l7.59-7.59L19 8l-9 9z"/>
|
||||
</svg>
|
||||
<span>Docs</span>
|
||||
</a>
|
||||
<a href="https://twitter.com/unclecode" target="_blank" class="social-link">
|
||||
<svg viewBox="0 0 24 24" width="16" height="16">
|
||||
<path fill="currentColor" d="M18.244 2.25h3.308l-7.227 8.26 8.502 11.24H16.17l-5.214-6.817L4.99 21.75H1.68l7.73-8.835L1.254 2.25H8.08l4.713 6.231zm-1.161 17.52h1.833L7.084 4.126H5.117z"/>
|
||||
</svg>
|
||||
<span>@unclecode</span>
|
||||
</a>
|
||||
<a href="https://discord.gg/jP8KfhDhyN" target="_blank" class="social-link">
|
||||
<svg viewBox="0 0 24 24" width="16" height="16">
|
||||
<path fill="currentColor" d="M19.27 5.33C17.94 4.71 16.5 4.26 15 4a.09.09 0 00-.07.03c-.18.33-.39.76-.53 1.09a16.09 16.09 0 00-4.8 0c-.14-.34-.35-.76-.54-1.09-.01-.02-.04-.03-.07-.03-1.5.26-2.93.71-4.27 1.33-.01 0-.02.01-.03.02-2.72 4.07-3.47 8.03-3.1 11.95 0 .02.01.04.03.05 1.8 1.32 3.53 2.12 5.24 2.65.03.01.06 0 .07-.02.4-.55.76-1.13 1.07-1.74.02-.04 0-.08-.04-.09-.57-.22-1.11-.48-1.64-.78-.04-.02-.04-.08-.01-.11.11-.08.22-.17.33-.25.02-.02.05-.02.07-.01 3.44 1.57 7.15 1.57 10.55 0 .02-.01.05-.01.07.01.11.09.22.17.33.26.04.03.04.09-.01.11-.52.31-1.07.56-1.64.78-.04.01-.05.06-.04.09.32.61.68 1.19 1.07 1.74.03.01.06.02.09.01 1.72-.53 3.45-1.33 5.25-2.65.02-.01.03-.03.03-.05.44-4.53-.73-8.46-3.1-11.95-.01-.01-.02-.02-.04-.02zM8.52 14.91c-1.03 0-1.89-.95-1.89-2.12s.84-2.12 1.89-2.12c1.06 0 1.9.96 1.89 2.12 0 1.17-.84 2.12-1.89 2.12zm6.97 0c-1.03 0-1.89-.95-1.89-2.12s.84-2.12 1.89-2.12c1.06 0 1.9.96 1.89 2.12 0 1.17-.83 2.12-1.89 2.12z"/>
|
||||
</svg>
|
||||
<span>Discord</span>
|
||||
</a>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
|
||||
<script src="popup.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,146 @@
|
||||
// Popup script for Crawl4AI Assistant
|
||||
let activeMode = null;
|
||||
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
// Fetch GitHub stars
|
||||
fetchGitHubStars();
|
||||
|
||||
// Check current state
|
||||
chrome.storage.local.get(['captureMode', 'captureStats'], (data) => {
|
||||
if (data.captureMode) {
|
||||
activeMode = data.captureMode;
|
||||
showActiveSession(data.captureStats || {});
|
||||
}
|
||||
});
|
||||
|
||||
// Mode buttons
|
||||
document.getElementById('schema-mode').addEventListener('click', () => {
|
||||
startSchemaCapture();
|
||||
});
|
||||
|
||||
document.getElementById('script-mode').addEventListener('click', () => {
|
||||
startScriptCapture();
|
||||
});
|
||||
|
||||
document.getElementById('c2c-mode').addEventListener('click', () => {
|
||||
startClick2Crawl();
|
||||
});
|
||||
|
||||
// Session actions
|
||||
document.getElementById('generate-code').addEventListener('click', () => {
|
||||
generateCode();
|
||||
});
|
||||
|
||||
document.getElementById('stop-capture').addEventListener('click', () => {
|
||||
stopCapture();
|
||||
});
|
||||
});
|
||||
|
||||
async function fetchGitHubStars() {
|
||||
try {
|
||||
const response = await fetch('https://api.github.com/repos/unclecode/crawl4ai');
|
||||
const data = await response.json();
|
||||
const stars = data.stargazers_count;
|
||||
|
||||
// Format the number (e.g., 1.2k)
|
||||
let formattedStars;
|
||||
if (stars >= 1000) {
|
||||
formattedStars = (stars / 1000).toFixed(1) + 'k';
|
||||
} else {
|
||||
formattedStars = stars.toString();
|
||||
}
|
||||
|
||||
document.getElementById('stars-count').textContent = `⭐ ${formattedStars}`;
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch GitHub stars:', error);
|
||||
document.getElementById('stars-count').textContent = '⭐ 2k+';
|
||||
}
|
||||
}
|
||||
|
||||
function startSchemaCapture() {
|
||||
chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
|
||||
chrome.tabs.sendMessage(tabs[0].id, {
|
||||
action: 'startSchemaCapture'
|
||||
}, (response) => {
|
||||
if (response && response.success) {
|
||||
// Close the popup to let user interact with the page
|
||||
window.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function startScriptCapture() {
|
||||
chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
|
||||
chrome.tabs.sendMessage(tabs[0].id, {
|
||||
action: 'startScriptCapture'
|
||||
}, (response) => {
|
||||
if (response && response.success) {
|
||||
// Close the popup to let user interact with the page
|
||||
window.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function startClick2Crawl() {
|
||||
chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
|
||||
chrome.tabs.sendMessage(tabs[0].id, {
|
||||
action: 'startClick2Crawl'
|
||||
}, (response) => {
|
||||
if (response && response.success) {
|
||||
// Close the popup to let user interact with the page
|
||||
window.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function showActiveSession(stats) {
|
||||
document.querySelector('.mode-selector').style.display = 'none';
|
||||
document.getElementById('active-session').classList.remove('hidden');
|
||||
|
||||
updateSessionStats(stats);
|
||||
}
|
||||
|
||||
function updateSessionStats(stats) {
|
||||
document.getElementById('container-status').textContent =
|
||||
stats.container ? 'Selected ✓' : 'Not selected';
|
||||
document.getElementById('fields-count').textContent = stats.fields || 0;
|
||||
|
||||
// Enable generate button if we have container and fields
|
||||
document.getElementById('generate-code').disabled =
|
||||
!stats.container || stats.fields === 0;
|
||||
}
|
||||
|
||||
function generateCode() {
|
||||
chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
|
||||
chrome.tabs.sendMessage(tabs[0].id, {
|
||||
action: 'generateCode'
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function stopCapture() {
|
||||
chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
|
||||
chrome.tabs.sendMessage(tabs[0].id, {
|
||||
action: 'stopCapture'
|
||||
}, () => {
|
||||
// Reset UI
|
||||
document.querySelector('.mode-selector').style.display = 'flex';
|
||||
document.getElementById('active-session').classList.add('hidden');
|
||||
activeMode = null;
|
||||
|
||||
// Clear storage
|
||||
chrome.storage.local.remove(['captureMode', 'captureStats']);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Listen for stats updates from content script
|
||||
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
|
||||
if (message.action === 'updateStats') {
|
||||
updateSessionStats(message.stats);
|
||||
chrome.storage.local.set({ captureStats: message.stats });
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user