chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Deploy script for claude-code-templates
|
||||
# Deploys the Astro dashboard which serves both www.aitmpl.com and app.aitmpl.com
|
||||
#
|
||||
# Required env vars (from .env):
|
||||
# VERCEL_ORG_ID, VERCEL_DASHBOARD_PROJECT_ID
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/deploy.sh # Deploy www + app.aitmpl.com
|
||||
# ./scripts/deploy.sh dashboard # Same as above (backwards compat)
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
# Load .env if present
|
||||
if [[ -f "$REPO_ROOT/.env" ]]; then
|
||||
set -a
|
||||
source "$REPO_ROOT/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# Validate required vars
|
||||
for var in VERCEL_ORG_ID VERCEL_DASHBOARD_PROJECT_ID; do
|
||||
if [[ -z "${!var:-}" ]]; then
|
||||
echo "Error: $var is not set. Add it to .env" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
deploy() {
|
||||
echo "=> Deploying www.aitmpl.com + app.aitmpl.com (Astro dashboard)..."
|
||||
VERCEL_ORG_ID="$VERCEL_ORG_ID" \
|
||||
VERCEL_PROJECT_ID="$VERCEL_DASHBOARD_PROJECT_ID" \
|
||||
npx vercel --prod --yes --cwd "$REPO_ROOT"
|
||||
echo "=> Deployed successfully."
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
""|dashboard|all)
|
||||
deploy
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 [dashboard]"
|
||||
echo ""
|
||||
echo " Deploys the Astro dashboard serving www.aitmpl.com + app.aitmpl.com"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,46 @@
|
||||
const express = require('express');
|
||||
const path = require('path');
|
||||
const app = express();
|
||||
const port = 3001;
|
||||
|
||||
// Serve static files from docs directory
|
||||
app.use(express.static('docs'));
|
||||
|
||||
// Handle filter routes - redirect to index.html
|
||||
const filterRoutes = ['agents', 'commands', 'settings', 'hooks', 'mcps', 'templates'];
|
||||
filterRoutes.forEach(filter => {
|
||||
app.get(`/${filter}`, (req, res) => {
|
||||
res.sendFile(path.join(__dirname, 'docs', 'index.html'));
|
||||
});
|
||||
});
|
||||
|
||||
// Handle component routes
|
||||
app.get('/component/:type/:name', (req, res) => {
|
||||
res.sendFile(path.join(__dirname, 'docs', 'component.html'));
|
||||
});
|
||||
|
||||
// Handle blog routes
|
||||
app.get('/blog/*', (req, res) => {
|
||||
const blogPath = req.path.replace('/blog', '');
|
||||
res.sendFile(path.join(__dirname, 'docs', 'blog', blogPath, 'index.html'), (err) => {
|
||||
if (err) {
|
||||
res.status(404).send('Blog post not found');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Default route
|
||||
app.get('/', (req, res) => {
|
||||
res.sendFile(path.join(__dirname, 'docs', 'index.html'));
|
||||
});
|
||||
|
||||
app.listen(port, () => {
|
||||
console.log(`Development server running at http://localhost:${port}`);
|
||||
console.log('Testing URLs:');
|
||||
console.log(`- http://localhost:${port}/ (agents)`);
|
||||
console.log(`- http://localhost:${port}/mcps`);
|
||||
console.log(`- http://localhost:${port}/commands`);
|
||||
console.log(`- http://localhost:${port}/settings`);
|
||||
console.log(`- http://localhost:${port}/hooks`);
|
||||
console.log(`- http://localhost:${port}/templates`);
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate a lightweight agents API endpoint from components.json
|
||||
This creates docs/api/agents.json for the CLI tool to use
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
def generate_agents_api():
|
||||
"""Generate the agents API file from components.json"""
|
||||
|
||||
# Read the components.json file
|
||||
components_path = 'docs/components.json'
|
||||
output_path = 'docs/api/agents.json'
|
||||
|
||||
if not os.path.exists(components_path):
|
||||
print(f"Error: {components_path} not found")
|
||||
return False
|
||||
|
||||
try:
|
||||
with open(components_path, 'r', encoding='utf-8') as f:
|
||||
components_data = json.load(f)
|
||||
|
||||
# Extract agents and format them for the API
|
||||
agents = []
|
||||
if 'agents' in components_data:
|
||||
for agent in components_data['agents']:
|
||||
# Extract category from path
|
||||
path_parts = agent['path'].split('/')
|
||||
category = path_parts[0] if len(path_parts) > 1 else 'root'
|
||||
name = path_parts[-1]
|
||||
|
||||
# Remove .md extension from name if present
|
||||
if name.endswith('.md'):
|
||||
name = name[:-3]
|
||||
|
||||
agents.append({
|
||||
'name': name,
|
||||
'path': agent['path'].replace('.md', ''), # Remove .md from path too
|
||||
'category': category,
|
||||
'description': agent.get('description', '')[:100] # Truncate description for size
|
||||
})
|
||||
|
||||
# Create output directory if it doesn't exist
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
|
||||
# Write the API file
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump({
|
||||
'agents': agents,
|
||||
'version': '1.0.0',
|
||||
'total': len(agents)
|
||||
}, f, indent=2)
|
||||
|
||||
print(f"✅ Generated agents API with {len(agents)} agents")
|
||||
print(f"📄 Output: {output_path}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error generating agents API: {e}")
|
||||
return False
|
||||
|
||||
if __name__ == '__main__':
|
||||
generate_agents_api()
|
||||
Executable
+315
@@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate blog cover images using Google AI (Imagen API).
|
||||
|
||||
Reads blog articles from ../docs/blog/blog-articles.json and generates
|
||||
cover images for articles that don't have images in ../docs/blog/assets/
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import base64
|
||||
import requests
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def check_google_api_key():
|
||||
"""Check for Google API key in environment."""
|
||||
api_key = os.getenv('GOOGLE_API_KEY')
|
||||
|
||||
if not api_key:
|
||||
# Try loading from .env file
|
||||
env_file = Path('.env')
|
||||
if env_file.exists():
|
||||
with open(env_file, 'r') as f:
|
||||
for line in f:
|
||||
if line.startswith('GOOGLE_API_KEY='):
|
||||
api_key = line.split('=', 1)[1].strip().strip('"').strip("'")
|
||||
break
|
||||
|
||||
if not api_key:
|
||||
print("❌ Error: GOOGLE_API_KEY not found!")
|
||||
print("\nPlease set the environment variable:")
|
||||
print("export GOOGLE_API_KEY=your-api-key-here")
|
||||
print("\nOr create a .env file with:")
|
||||
print("GOOGLE_API_KEY=your-api-key-here")
|
||||
sys.exit(1)
|
||||
|
||||
return api_key
|
||||
|
||||
|
||||
def generate_blog_image(title, description, component_type, component_name, install_command, output_path, api_key):
|
||||
"""
|
||||
Generate a blog cover image using Google's Imagen API via AI Studio.
|
||||
|
||||
Args:
|
||||
title: Article title
|
||||
description: Article description
|
||||
component_type: Type of component (Agent, MCP, Skill, etc.)
|
||||
component_name: Name of the component
|
||||
install_command: Installation command
|
||||
output_path: Path to save the generated image
|
||||
api_key: Google API key
|
||||
"""
|
||||
# Split install command into two lines for better readability
|
||||
if "--agent" in install_command:
|
||||
cmd_parts = install_command.split("--agent ")
|
||||
cmd_line1 = cmd_parts[0].strip()
|
||||
cmd_line2 = "--agent " + cmd_parts[1]
|
||||
elif "--mcp" in install_command:
|
||||
cmd_parts = install_command.split("--mcp ")
|
||||
cmd_line1 = cmd_parts[0].strip()
|
||||
cmd_line2 = "--mcp " + cmd_parts[1]
|
||||
elif "--skill" in install_command:
|
||||
cmd_parts = install_command.split("--skill ")
|
||||
cmd_line1 = cmd_parts[0].strip()
|
||||
cmd_line2 = "--skill " + cmd_parts[1]
|
||||
else:
|
||||
cmd_line1 = install_command
|
||||
cmd_line2 = ""
|
||||
|
||||
# Simplify the command display to avoid text rendering issues
|
||||
# Show just the essential part
|
||||
if cmd_line2:
|
||||
simple_cmd = cmd_line2.strip() # Just show "--agent folder/name" etc
|
||||
else:
|
||||
simple_cmd = cmd_line1
|
||||
|
||||
# Create detailed prompt similar to supabase-claude-code-templates-cover.png
|
||||
prompt = f"""Create a professional blog cover image with this EXACT layout and text:
|
||||
|
||||
LEFT SIDE (40% width):
|
||||
- Black background (#000000)
|
||||
- Text exactly as "CLAUDE CODE TEMPLATES" in pixelated/retro orange font (#F97316)
|
||||
- Font style: Bold, blocky, retro gaming aesthetic similar to arcade game fonts
|
||||
- Stacked vertically with equal spacing between words
|
||||
- Centered vertically on left side
|
||||
|
||||
CENTER:
|
||||
- Vertical line divider in dark gray (#333333) 2px width
|
||||
- Full height of image from top to bottom
|
||||
|
||||
RIGHT SIDE (60% width):
|
||||
- Black background (#000000)
|
||||
- At top: Small badge with text "{component_type}" in solid orange rectangle (#F97316) with rounded corners, black text inside
|
||||
- Below badge: Large bold white text saying "{component_name}"
|
||||
- At bottom: Small gray monospace text showing: "{simple_cmd}"
|
||||
|
||||
CRITICAL TEXT RENDERING REQUIREMENTS:
|
||||
- All text must be perfectly readable and not corrupted
|
||||
- Use clear, legible fonts
|
||||
- Ensure proper spacing so text doesn't overlap or get cut off
|
||||
- The installation command at bottom should be clearly visible
|
||||
|
||||
Overall specifications:
|
||||
- Exact dimensions: 1200x675 pixels (16:9 aspect ratio)
|
||||
- Background: Pure black (#000000)
|
||||
- Primary accent color: Orange (#F97316)
|
||||
- Text colors: White (#FFFFFF) for component name, Gray (#999999) for command
|
||||
- Minimalist, professional design
|
||||
- No decorative elements, gradients, or patterns - just text and divider
|
||||
- Clean, modern tech aesthetic similar to terminal/CLI interfaces
|
||||
|
||||
The layout divides the image into two sections with a vertical line: left side shows "CLAUDE CODE TEMPLATES" in retro orange font, right side shows component type, name, and install command."""
|
||||
|
||||
print(f"🎨 Generating image for: {title}")
|
||||
print(f"📝 Prompt length: {len(prompt)} chars")
|
||||
|
||||
# Google AI Nano Banana (gemini-2.5-flash-image) endpoint
|
||||
url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-image:generateContent"
|
||||
|
||||
payload = {
|
||||
"contents": [
|
||||
{
|
||||
"parts": [
|
||||
{
|
||||
"text": prompt
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
headers = {
|
||||
"x-goog-api-key": api_key,
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.post(url, headers=headers, json=payload)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ API Error ({response.status_code}): {response.text}")
|
||||
return False
|
||||
|
||||
result = response.json()
|
||||
|
||||
# Extract image from Nano Banana response
|
||||
if "candidates" in result and len(result["candidates"]) > 0:
|
||||
candidate = result["candidates"][0]
|
||||
|
||||
if "content" in candidate and "parts" in candidate["content"]:
|
||||
parts = candidate["content"]["parts"]
|
||||
|
||||
# Find the inline data part with the image
|
||||
for part in parts:
|
||||
if "inlineData" in part:
|
||||
inline_data = part["inlineData"]
|
||||
|
||||
# Extract base64 data
|
||||
if "data" in inline_data:
|
||||
image_data = base64.b64decode(inline_data["data"])
|
||||
|
||||
# Save image
|
||||
with open(output_path, 'wb') as f:
|
||||
f.write(image_data)
|
||||
|
||||
print(f"✅ Image saved to: {output_path}")
|
||||
return True
|
||||
|
||||
print(f"⚠️ No inline data found in response parts")
|
||||
return False
|
||||
else:
|
||||
print(f"⚠️ Unexpected response structure: {result}")
|
||||
return False
|
||||
else:
|
||||
print(f"❌ No candidates in response: {result}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error generating image: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to generate blog images."""
|
||||
# Get Google API key
|
||||
api_key = check_google_api_key()
|
||||
|
||||
# Load blog articles
|
||||
blog_json_path = Path(__file__).parent.parent / "docs" / "blog" / "blog-articles.json"
|
||||
|
||||
if not blog_json_path.exists():
|
||||
print(f"❌ Error: Blog articles file not found: {blog_json_path}")
|
||||
sys.exit(1)
|
||||
|
||||
with open(blog_json_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
articles = data.get("articles", [])
|
||||
assets_dir = blog_json_path.parent / "assets"
|
||||
|
||||
# Create assets directory if it doesn't exist
|
||||
assets_dir.mkdir(exist_ok=True)
|
||||
|
||||
print(f"\n📚 Found {len(articles)} articles")
|
||||
print(f"📁 Assets directory: {assets_dir}\n")
|
||||
|
||||
# Filter articles that need images generated (hosted on aitmpl.com/blog/assets/)
|
||||
articles_needing_images = []
|
||||
for article in articles:
|
||||
image_url = article.get("image", "")
|
||||
if "aitmpl.com/blog/assets/" in image_url and "-cover.png" in image_url:
|
||||
# Extract filename from URL
|
||||
filename = image_url.split("/")[-1]
|
||||
output_path = assets_dir / filename
|
||||
|
||||
if not output_path.exists():
|
||||
articles_needing_images.append({
|
||||
"article": article,
|
||||
"filename": filename,
|
||||
"output_path": output_path
|
||||
})
|
||||
else:
|
||||
print(f"⏭️ Skipping {filename} (already exists)")
|
||||
|
||||
if not articles_needing_images:
|
||||
print("\n✅ All blog images already exist!")
|
||||
return
|
||||
|
||||
print(f"\n🎨 Generating {len(articles_needing_images)} images...\n")
|
||||
|
||||
# Generate images
|
||||
success_count = 0
|
||||
for item in articles_needing_images:
|
||||
article = item["article"]
|
||||
filename = item["filename"]
|
||||
output_path = item["output_path"]
|
||||
|
||||
# Extract component information from article ID
|
||||
article_id = article.get("id", "")
|
||||
|
||||
# Determine component type and name based on article category
|
||||
category = article.get("category", "")
|
||||
|
||||
if "agent" in article_id.lower() or category == "Agents":
|
||||
component_type = "AGENT"
|
||||
# Extract agent name from ID (e.g., frontend-developer-agent -> frontend-developer)
|
||||
agent_name = article_id.replace("-agent", "")
|
||||
component_name = agent_name.replace("-", " ").title()
|
||||
|
||||
# Find actual agent path in components/agents/
|
||||
import subprocess
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["find", "cli-tool/components/agents", "-name", f"{agent_name}.md", "-type", "f"],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
agent_path = result.stdout.strip()
|
||||
if agent_path:
|
||||
# Extract folder/name from path (e.g., development-team/frontend-developer)
|
||||
parts = agent_path.split("/agents/")[1].replace(".md", "")
|
||||
install_command = f"npx claude-code-templates@latest --agent {parts}"
|
||||
else:
|
||||
install_command = f"npx claude-code-templates@latest --agent {agent_name}"
|
||||
except:
|
||||
install_command = f"npx claude-code-templates@latest --agent {agent_name}"
|
||||
elif "mcp" in article_id.lower() or category == "MCP":
|
||||
component_type = "MCP"
|
||||
component_name = article_id.replace("-mcp", "").replace("-", " ").title()
|
||||
install_command = f"npx claude-code-templates@latest --mcp {article_id.replace('-mcp', '')}"
|
||||
elif "skill" in article_id.lower() or category == "Skills":
|
||||
component_type = "SKILL"
|
||||
component_name = article_id.replace("-skill", "").replace("-", " ").title()
|
||||
install_command = f"npx claude-code-templates@latest --skill {article_id}"
|
||||
elif "sandbox" in article_id.lower() or "e2b" in article_id.lower():
|
||||
component_type = "SANDBOX"
|
||||
component_name = article_id.replace("-", " ").title()
|
||||
install_command = f"npx claude-code-templates@latest --sandbox e2b"
|
||||
else:
|
||||
component_type = "COMPONENT"
|
||||
component_name = article_id.replace("-", " ").title()
|
||||
install_command = f"npx claude-code-templates@latest"
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📄 Article: {article['title']}")
|
||||
print(f"🏷️ Type: {component_type}")
|
||||
print(f"📦 Component: {component_name}")
|
||||
print(f"💻 Command: {install_command}")
|
||||
print(f"💾 Output: {filename}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
success = generate_blog_image(
|
||||
title=article["title"],
|
||||
description=article["description"],
|
||||
component_type=component_type,
|
||||
component_name=component_name,
|
||||
install_command=install_command,
|
||||
output_path=str(output_path),
|
||||
api_key=api_key
|
||||
)
|
||||
|
||||
if success:
|
||||
success_count += 1
|
||||
|
||||
print() # Extra newline for readability
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"✅ Successfully generated {success_count}/{len(articles_needing_images)} images")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script to generate blog images using Google Gemini 2.5 Flash Image (nano banana)
|
||||
Generates banners and workflow diagrams for Claude Code component blogs
|
||||
"""
|
||||
|
||||
import os
|
||||
import base64
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
from google import genai
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv(Path(__file__).parent.parent / '.env')
|
||||
|
||||
# Configuration
|
||||
API_KEY = os.environ.get("GOOGLE_API_KEY")
|
||||
if not API_KEY:
|
||||
raise ValueError("GOOGLE_API_KEY not found in environment variables. Check your .env file.")
|
||||
|
||||
OUTPUT_DIR = Path(__file__).parent.parent / "docs/blog/assets"
|
||||
MODEL = "gemini-2.0-flash-exp-image-generation" # Using Gemini 2.0 Flash Exp with image generation
|
||||
|
||||
# Blog definitions with prompts
|
||||
BLOGS = [
|
||||
{
|
||||
"id": "frontend-developer-agent",
|
||||
"title": "Claude Code Frontend Developer Agent: Complete 2025 Tutorial",
|
||||
"banner_prompt": """Create a professional tech banner image with these elements:
|
||||
- Dark terminal/coding background with subtle grid pattern
|
||||
- Text overlay in bright green monospace font: 'Frontend Developer Agent'
|
||||
- Subtitle in smaller text: 'Complete 2025 Tutorial - Claude Code'
|
||||
- Include subtle React, Vue, and Next.js logo icons in corners
|
||||
- Modern code editor aesthetic with syntax highlighting in background
|
||||
- Color scheme: Dark (#1a1a1a) with neon green (#00ff41) accents
|
||||
- Professional, clean, minimalist design
|
||||
- 1200x630 px banner format""",
|
||||
|
||||
"diagram_prompt": """Create a simple technical flowchart diagram:
|
||||
Title: Frontend Agent Workflow
|
||||
Flow: User Input → Frontend Agent Analysis → Framework Detection (React/Vue/Next) → Code Generation → Component Creation → Testing → Output
|
||||
Style: Clean minimal flowchart with boxes and arrows
|
||||
Colors: Terminal theme with green text on dark background
|
||||
Professional documentation style"""
|
||||
}
|
||||
]
|
||||
|
||||
def create_output_dir():
|
||||
"""Create output directory if it doesn't exist"""
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
print(f"✓ Output directory ready: {OUTPUT_DIR}")
|
||||
|
||||
|
||||
def generate_image_with_gemini(prompt, output_path):
|
||||
"""Generate image using Gemini with native image generation"""
|
||||
try:
|
||||
client = genai.Client(api_key=API_KEY)
|
||||
|
||||
print(f" Generating with {MODEL}...")
|
||||
|
||||
# Use generate_content for Gemini models with image generation
|
||||
response = client.models.generate_content(
|
||||
model=MODEL,
|
||||
contents=prompt
|
||||
)
|
||||
|
||||
# The response should contain image data
|
||||
if hasattr(response, 'parts') and response.parts:
|
||||
for part in response.parts:
|
||||
if hasattr(part, 'inline_data') and part.inline_data:
|
||||
# Save the image
|
||||
with open(output_path, 'wb') as f:
|
||||
f.write(part.inline_data.data)
|
||||
print(f" ✓ Saved: {output_path}")
|
||||
return True
|
||||
|
||||
print(f" ✗ No image data in response")
|
||||
print(f" Response: {response}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f" ✗ Error: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
def test_single_image():
|
||||
"""Test with a single image first"""
|
||||
create_output_dir()
|
||||
|
||||
blog = BLOGS[0]
|
||||
print(f"\n🎨 Testing image generation for: {blog['title']}\n")
|
||||
|
||||
# Test banner
|
||||
banner_path = os.path.join(OUTPUT_DIR, f"{blog['id']}-cover-test.png")
|
||||
print("Testing Banner Generation:")
|
||||
success = generate_image_with_gemini(blog["banner_prompt"], banner_path)
|
||||
|
||||
if success:
|
||||
print(f"\n✅ Success! Test image saved to: {banner_path}")
|
||||
else:
|
||||
print(f"\n❌ Failed to generate image. Check API key and model availability.")
|
||||
print(f"\nTip: The API key might need Imagen 4 enabled or use a different approach.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_single_image()
|
||||
@@ -0,0 +1,769 @@
|
||||
"""
|
||||
Generate claude-jobs.json by scraping free job sources for Claude Code positions.
|
||||
|
||||
Sources (all free, no API keys required):
|
||||
1. HN Firebase API - "Who is Hiring" thread comments
|
||||
2. HN Algolia API - Search across multiple months
|
||||
3. RemoteOK API - JSON job feed
|
||||
4. WeWorkRemotely RSS - Programming jobs feed
|
||||
5. Anthropic Careers - Greenhouse API (jobs mentioning Claude Code)
|
||||
|
||||
Output: docs/claude-jobs.json
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from html import unescape
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.parse import quote_plus
|
||||
from urllib.error import URLError
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Utilities
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def fetch_json(url, timeout=15):
|
||||
"""Fetch JSON from a URL."""
|
||||
req = Request(url, headers={"User-Agent": "claude-code-templates/1.0"})
|
||||
try:
|
||||
with urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
except Exception as e:
|
||||
print(f" [warn] fetch_json failed for {url[:80]}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_text(url, timeout=15):
|
||||
"""Fetch raw text/XML from a URL."""
|
||||
req = Request(url, headers={"User-Agent": "claude-code-templates/1.0"})
|
||||
try:
|
||||
with urlopen(req, timeout=timeout) as resp:
|
||||
return resp.read().decode("utf-8")
|
||||
except Exception as e:
|
||||
print(f" [warn] fetch_text failed for {url[:80]}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def strip_html(html_text):
|
||||
"""Remove HTML tags, decode entities, and fix mojibake."""
|
||||
if not html_text:
|
||||
return ""
|
||||
text = re.sub(r"<[^>]+>", " ", html_text)
|
||||
text = unescape(text)
|
||||
# Fix common UTF-8 mojibake (double-decoded characters)
|
||||
mojibake_map = {
|
||||
"\u00e2\u0080\u0099": "\u2019", # right single quote '
|
||||
"\u00e2\u0080\u009c": "\u201c", # left double quote "
|
||||
"\u00e2\u0080\u009d": "\u201d", # right double quote "
|
||||
"\u00e2\u0080\u0094": "\u2014", # em dash —
|
||||
"\u00e2\u0080\u0093": "\u2013", # en dash –
|
||||
"\u00e2\u0080\u00a6": "\u2026", # ellipsis …
|
||||
"\u00c2\u00a0": " ", # non-breaking space
|
||||
}
|
||||
for bad, good in mojibake_map.items():
|
||||
text = text.replace(bad, good)
|
||||
# Catch remaining mojibake patterns: ’ â€" â€" etc.
|
||||
text = re.sub(r"\u00e2\u0080.", lambda m: "'", text)
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
return text
|
||||
|
||||
|
||||
def is_claude_related(text):
|
||||
"""Check if text mentions Claude Code or related tools."""
|
||||
if not text:
|
||||
return False
|
||||
t = text.lower()
|
||||
keywords = [
|
||||
"claude code", "claude-code", "anthropic claude", "claude ai",
|
||||
"claude coder", "using claude", "claude experience",
|
||||
]
|
||||
if any(kw in t for kw in keywords):
|
||||
return True
|
||||
# "claude" alone needs job context to avoid false positives
|
||||
if "claude" in t:
|
||||
job_words = ["hiring", "position", "engineer", "developer", "role",
|
||||
"stack", "workflow", "tool", "cursor"]
|
||||
return any(w in t for w in job_words)
|
||||
return False
|
||||
|
||||
|
||||
def truncate(text, length=200):
|
||||
"""Truncate text at word boundary."""
|
||||
if not text or len(text) <= length:
|
||||
return text or ""
|
||||
return text[:length].rsplit(" ", 1)[0] + "..."
|
||||
|
||||
|
||||
def extract_salary(text):
|
||||
"""Extract salary string from text.
|
||||
|
||||
Only matches compensation-like amounts (≥$10k or with k/K suffix).
|
||||
Skips funding/revenue figures like '$37M raised' or '$300M+ revenue'.
|
||||
"""
|
||||
if not text:
|
||||
return ""
|
||||
# First, remove sentences about funding/revenue to avoid false positives
|
||||
cleaned = re.sub(r"\$[\d,.]+\s*[MBmb](?:illion)?[^.]*(?:\.|$)", "", text)
|
||||
cleaned = re.sub(r"(?:raised|revenue|funding|valuation|ARR)[^.]*\$[\d,.]+[^.]*(?:\.|$)", "", cleaned, flags=re.IGNORECASE)
|
||||
patterns = [
|
||||
r"\$[\d,]+[kK]\s*[-\u2013]\s*\$?[\d,]+[kK]", # $120k-$150k, $120k-150k
|
||||
r"\$[\d,]+[kK](?:\+|\s*\/\s*(?:yr|year))?", # $150k+, $150k/yr
|
||||
r"\$[\d,]{3,}\s*[-\u2013]\s*\$?[\d,]{3,}", # $120,000-$150,000
|
||||
r"\$[\d,]+[kK]?\s*\/\s*(?:yr|year)", # $120,000/yr
|
||||
r"[\d,]+[kK]\s*[-\u2013]\s*[\d,]+[kK]", # 120k-150k
|
||||
r"\u20ac[\d,]+[kK]?\s*[-\u2013]\s*\u20ac?[\d,]+[kK]?", # EUR
|
||||
]
|
||||
for pat in patterns:
|
||||
m = re.search(pat, cleaned)
|
||||
if m:
|
||||
val = m.group(0)
|
||||
# Validate: must represent a realistic salary
|
||||
nums = re.findall(r"[\d,]+", val)
|
||||
has_k = "k" in val.lower()
|
||||
if nums:
|
||||
first_num = int(nums[0].replace(",", ""))
|
||||
# With 'k' suffix: must be >= 30 (i.e. $30k+)
|
||||
if has_k and first_num < 30:
|
||||
continue
|
||||
# Without 'k': must be >= 30,000 (raw dollar amounts)
|
||||
# Values like $100-140 or $150-300 are ambiguous/truncated
|
||||
if not has_k and first_num < 1000:
|
||||
continue
|
||||
return val
|
||||
return ""
|
||||
|
||||
|
||||
def extract_urls(text):
|
||||
"""Extract URLs from text."""
|
||||
return re.findall(r"https?://[^\s<>\"']+", text or "")
|
||||
|
||||
|
||||
def company_icon(name):
|
||||
"""Best-effort favicon URL for a company."""
|
||||
known = {
|
||||
"anthropic": "https://www.anthropic.com/favicon.ico",
|
||||
"google": "https://www.google.com/favicon.ico",
|
||||
"microsoft": "https://www.microsoft.com/favicon.ico",
|
||||
"meta": "https://www.facebook.com/favicon.ico",
|
||||
"stripe": "https://stripe.com/favicon.ico",
|
||||
"github": "https://github.com/favicon.ico",
|
||||
"vercel": "https://vercel.com/favicon.ico",
|
||||
"supabase": "https://supabase.com/favicon.ico",
|
||||
"openai": "https://openai.com/favicon.ico",
|
||||
"shopify": "https://shopify.com/favicon.ico",
|
||||
"notion": "https://notion.so/favicon.ico",
|
||||
"figma": "https://figma.com/favicon.ico",
|
||||
}
|
||||
lower = name.lower().strip()
|
||||
for key, icon in known.items():
|
||||
if key in lower:
|
||||
return icon
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HN Comment Parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def parse_hn_comment(comment_data):
|
||||
"""Parse a HN 'Who is Hiring' comment into a structured job dict.
|
||||
|
||||
HN hiring comments typically follow:
|
||||
CompanyName | Location | Remote | Salary | URL
|
||||
Description paragraph(s)...
|
||||
"""
|
||||
text = comment_data.get("text", "")
|
||||
if not text:
|
||||
return None
|
||||
|
||||
clean = strip_html(text)
|
||||
if not is_claude_related(clean):
|
||||
return None
|
||||
|
||||
comment_id = comment_data.get("id", "")
|
||||
posted_ts = comment_data.get("time", 0)
|
||||
posted_at = datetime.fromtimestamp(posted_ts, tz=timezone.utc).isoformat() if posted_ts else ""
|
||||
|
||||
# --- Parse first line (pipe-delimited header) ---
|
||||
# Split on <p> to get paragraphs
|
||||
paragraphs = re.split(r"<p>", text)
|
||||
header_html = paragraphs[0] if paragraphs else ""
|
||||
header = strip_html(header_html)
|
||||
|
||||
# Try pipe-delimited: "Company | Location | Remote | ..."
|
||||
parts = [p.strip() for p in header.split("|")]
|
||||
|
||||
company = parts[0] if parts else "Unknown"
|
||||
# Clean company name (remove trailing URLs, etc.)
|
||||
company = re.sub(r"https?://\S+", "", company).strip()
|
||||
company = re.sub(r"\s*[-\u2013(].*", "", company).strip() if len(company) > 60 else company
|
||||
|
||||
# Location
|
||||
location = ""
|
||||
remote = False
|
||||
# Words that indicate a part is a role title, not a location
|
||||
role_words = {"engineer", "developer", "lead", "manager", "architect",
|
||||
"designer", "scientist", "analyst", "operations", "head",
|
||||
"director", "founder", "cto", "ceo", "vp ", "senior",
|
||||
"staff", "principal", "junior", "intern", "product",
|
||||
"technical", "native"}
|
||||
for part in parts[1:]:
|
||||
lower = part.lower().strip()
|
||||
# Skip parts that look like role titles
|
||||
if any(w in lower for w in role_words):
|
||||
continue
|
||||
if any(w in lower for w in ["remote", "anywhere", "distributed"]):
|
||||
remote = True
|
||||
if "only" in lower or "(" in lower or len(lower) > 6:
|
||||
location = part.strip()
|
||||
elif re.search(r"[A-Z][a-z]+", part) and not re.match(r"^\$", part.strip()):
|
||||
if not location and any(w in lower for w in [",", "city", "francisco", "york", "london",
|
||||
"berlin", "seattle", "austin", "chicago",
|
||||
"boston", "denver", "miami", "toronto",
|
||||
"europe", "us ", "u.s.", "worldwide"]):
|
||||
location = part.strip()
|
||||
elif not location and len(part.strip()) < 40:
|
||||
location = part.strip()
|
||||
|
||||
# Also check full text for remote indicators if not found in header
|
||||
if not remote:
|
||||
remote_in_body = any(w in clean.lower() for w in ["remote", "anywhere", "distributed", "work from home"])
|
||||
if remote_in_body:
|
||||
remote = True
|
||||
|
||||
if not location:
|
||||
location = "Remote" if remote else "On-site"
|
||||
elif remote and "remote" not in location.lower():
|
||||
location = f"{location} (Remote)"
|
||||
|
||||
# Salary
|
||||
salary = extract_salary(header)
|
||||
if not salary:
|
||||
salary = extract_salary(clean)
|
||||
|
||||
# Apply URL — first URL found in comment
|
||||
urls = extract_urls(text)
|
||||
apply_url = urls[0] if urls else f"https://news.ycombinator.com/item?id={comment_id}"
|
||||
|
||||
# Description — everything after header, truncated
|
||||
body_parts = paragraphs[1:] if len(paragraphs) > 1 else []
|
||||
description_html = " ".join(body_parts)
|
||||
description = truncate(strip_html(description_html), 300)
|
||||
if not description:
|
||||
description = truncate(clean, 300)
|
||||
|
||||
# Tags — extract tech keywords
|
||||
tags = extract_tech_tags(clean)
|
||||
|
||||
return {
|
||||
"id": f"hn-{comment_id}",
|
||||
"company": company[:80],
|
||||
"position": extract_position(clean, company),
|
||||
"location": location[:80],
|
||||
"remote": remote or "remote" in location.lower(),
|
||||
"salary": salary,
|
||||
"description": description,
|
||||
"applyUrl": apply_url,
|
||||
"source": "HackerNews",
|
||||
"sourceUrl": f"https://news.ycombinator.com/item?id={comment_id}",
|
||||
"postedAt": posted_at,
|
||||
"tags": tags,
|
||||
"companyIcon": company_icon(company),
|
||||
}
|
||||
|
||||
|
||||
def extract_position(text, company):
|
||||
"""Try to extract a job title from the text."""
|
||||
# Common patterns in HN posts
|
||||
patterns = [
|
||||
r"(?:hiring|looking for|seeking)\s+(?:a\s+)?([A-Z][A-Za-z\s/\-&]+(?:Engineer|Developer|Architect|Designer|Manager|Lead|Scientist|Analyst|Programmer))",
|
||||
r"((?:Senior|Staff|Principal|Lead|Junior|Mid[- ]?Level|Head of)\s+[A-Za-z\s/\-&]+(?:Engineer|Developer|Architect|Designer|Manager|Scientist))",
|
||||
r"((?:Full[- ]?Stack|Front[- ]?end|Back[- ]?end|DevOps|ML|AI|Platform|Infrastructure|Software|Product)\s+(?:Engineer|Developer|Architect|Manager))",
|
||||
]
|
||||
for pat in patterns:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
title = m.group(1).strip()
|
||||
# Don't return the company name as position
|
||||
if title.lower() != company.lower() and len(title) < 80:
|
||||
return title
|
||||
return "Software Engineer"
|
||||
|
||||
|
||||
def extract_tech_tags(text):
|
||||
"""Extract technology tags from text."""
|
||||
tech_keywords = {
|
||||
"react": "React", "next.js": "Next.js", "nextjs": "Next.js",
|
||||
"typescript": "TypeScript", "javascript": "JavaScript",
|
||||
"python": "Python", "rust": "Rust", "go ": "Go", "golang": "Go",
|
||||
"node.js": "Node.js", "nodejs": "Node.js",
|
||||
"postgresql": "PostgreSQL", "postgres": "PostgreSQL",
|
||||
"supabase": "Supabase", "firebase": "Firebase",
|
||||
"aws": "AWS", "gcp": "GCP", "azure": "Azure",
|
||||
"docker": "Docker", "kubernetes": "Kubernetes",
|
||||
"claude code": "Claude Code", "claude-code": "Claude Code",
|
||||
"cursor": "Cursor", "copilot": "Copilot",
|
||||
"react native": "React Native", "swift": "Swift",
|
||||
"kotlin": "Kotlin", "java ": "Java",
|
||||
"ruby": "Ruby", "rails": "Rails",
|
||||
"django": "Django", "flask": "Flask",
|
||||
"vue": "Vue.js", "angular": "Angular", "svelte": "Svelte",
|
||||
"tailwind": "Tailwind", "graphql": "GraphQL",
|
||||
"redis": "Redis", "mongodb": "MongoDB", "mysql": "MySQL",
|
||||
}
|
||||
found = []
|
||||
t = text.lower()
|
||||
for keyword, label in tech_keywords.items():
|
||||
if keyword in t and label not in found:
|
||||
found.append(label)
|
||||
return found[:10] # Cap at 10 tags
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source 1: HN Firebase API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def find_latest_hiring_threads():
|
||||
"""Find the latest 'Who is Hiring' thread IDs via Algolia."""
|
||||
print("[1/5] Finding latest HN 'Who is Hiring' threads...")
|
||||
url = (
|
||||
"https://hn.algolia.com/api/v1/search?"
|
||||
"query=%22who%20is%20hiring%22&tags=story&hitsPerPage=6"
|
||||
f"&numericFilters=created_at_i>{int(time.time()) - 86400 * 100}"
|
||||
)
|
||||
data = fetch_json(url)
|
||||
if not data:
|
||||
return []
|
||||
threads = []
|
||||
for hit in data.get("hits", []):
|
||||
title = (hit.get("title") or "").lower()
|
||||
if "who is hiring" in title and "freelancer" not in title and "wants to be hired" not in title:
|
||||
threads.append({
|
||||
"id": int(hit["objectID"]),
|
||||
"title": hit.get("title", ""),
|
||||
"date": hit.get("created_at", ""),
|
||||
})
|
||||
print(f" Found {len(threads)} hiring threads")
|
||||
return threads[:3] # Last 3 months
|
||||
|
||||
|
||||
def fetch_hn_thread_jobs(thread_id):
|
||||
"""Fetch all top-level comments from a HN thread and filter for Claude jobs."""
|
||||
print(f" Fetching thread {thread_id}...")
|
||||
thread_data = fetch_json(f"https://hacker-news.firebaseio.com/v0/item/{thread_id}.json")
|
||||
if not thread_data:
|
||||
return []
|
||||
|
||||
kid_ids = thread_data.get("kids", [])
|
||||
print(f" Thread has {len(kid_ids)} top-level comments, fetching...")
|
||||
|
||||
jobs = []
|
||||
|
||||
# Batch fetch comments with thread pool
|
||||
def fetch_comment(cid):
|
||||
return fetch_json(f"https://hacker-news.firebaseio.com/v0/item/{cid}.json", timeout=10)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=20) as executor:
|
||||
futures = {executor.submit(fetch_comment, cid): cid for cid in kid_ids}
|
||||
for future in as_completed(futures):
|
||||
comment = future.result()
|
||||
if comment and not comment.get("deleted") and not comment.get("dead"):
|
||||
job = parse_hn_comment(comment)
|
||||
if job:
|
||||
jobs.append(job)
|
||||
|
||||
print(f" Found {len(jobs)} Claude-related jobs in thread {thread_id}")
|
||||
return jobs
|
||||
|
||||
|
||||
def collect_hn_firebase():
|
||||
"""Collect jobs from HN Firebase API."""
|
||||
threads = find_latest_hiring_threads()
|
||||
all_jobs = []
|
||||
for thread in threads:
|
||||
jobs = fetch_hn_thread_jobs(thread["id"])
|
||||
# Tag jobs with the thread month
|
||||
for job in jobs:
|
||||
if not job.get("postedAt") and thread.get("date"):
|
||||
job["postedAt"] = thread["date"]
|
||||
all_jobs.extend(jobs)
|
||||
time.sleep(0.5)
|
||||
return all_jobs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source 2: HN Algolia API (supplementary search)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def collect_hn_algolia(existing_ids):
|
||||
"""Search HN Algolia for Claude Code mentions in hiring threads."""
|
||||
print("[2/5] Searching HN Algolia for additional Claude Code mentions...")
|
||||
jobs = []
|
||||
|
||||
search_terms = ["claude code", "claude-code"]
|
||||
for term in search_terms:
|
||||
url = (
|
||||
f"https://hn.algolia.com/api/v1/search?"
|
||||
f"query={quote_plus(term)}&tags=comment&hitsPerPage=50"
|
||||
f"&numericFilters=created_at_i>{int(time.time()) - 86400 * 100}"
|
||||
)
|
||||
data = fetch_json(url)
|
||||
if not data:
|
||||
continue
|
||||
|
||||
for hit in data.get("hits", []):
|
||||
hn_id = f"hn-{hit.get('objectID', '')}"
|
||||
if hn_id in existing_ids:
|
||||
continue
|
||||
|
||||
# Only include comments from hiring threads
|
||||
story_title = (hit.get("story_title") or "").lower()
|
||||
if "who is hiring" not in story_title:
|
||||
continue
|
||||
|
||||
comment_text = hit.get("comment_text", "")
|
||||
if not is_claude_related(strip_html(comment_text)):
|
||||
continue
|
||||
|
||||
# Build a minimal comment structure for the parser
|
||||
fake_comment = {
|
||||
"id": hit.get("objectID", ""),
|
||||
"text": comment_text,
|
||||
"time": hit.get("created_at_i", 0),
|
||||
}
|
||||
job = parse_hn_comment(fake_comment)
|
||||
if job:
|
||||
jobs.append(job)
|
||||
existing_ids.add(hn_id)
|
||||
|
||||
time.sleep(0.5)
|
||||
|
||||
print(f" Found {len(jobs)} additional jobs via Algolia")
|
||||
return jobs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source 3: RemoteOK API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def collect_remoteok():
|
||||
"""Collect jobs from RemoteOK API."""
|
||||
print("[3/5] Searching RemoteOK API...")
|
||||
data = fetch_json("https://remoteok.com/api")
|
||||
if not data:
|
||||
return []
|
||||
|
||||
jobs = []
|
||||
for item in data[1:]: # First element is metadata
|
||||
desc = item.get("description", "")
|
||||
tags_list = item.get("tags", [])
|
||||
tags_str = " ".join(tags_list) if isinstance(tags_list, list) else str(tags_list)
|
||||
full_text = f"{item.get('position', '')} {desc} {tags_str} {item.get('company', '')}"
|
||||
|
||||
if not is_claude_related(full_text):
|
||||
continue
|
||||
|
||||
slug = item.get("slug", item.get("id", ""))
|
||||
jobs.append({
|
||||
"id": f"rok-{slug}",
|
||||
"company": item.get("company", "Unknown"),
|
||||
"position": item.get("position", "Software Engineer"),
|
||||
"location": "Remote",
|
||||
"remote": True,
|
||||
"salary": extract_salary(f"{item.get('salary_min', '')} {item.get('salary_max', '')}"),
|
||||
"description": truncate(strip_html(desc), 300),
|
||||
"applyUrl": item.get("apply_url", "") or f"https://remoteok.com/remote-jobs/{slug}",
|
||||
"source": "RemoteOK",
|
||||
"sourceUrl": f"https://remoteok.com/remote-jobs/{slug}",
|
||||
"postedAt": item.get("date", ""),
|
||||
"tags": extract_tech_tags(full_text),
|
||||
"companyIcon": item.get("company_logo", "") or company_icon(item.get("company", "")),
|
||||
})
|
||||
|
||||
print(f" Found {len(jobs)} jobs from RemoteOK")
|
||||
return jobs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source 4: WeWorkRemotely RSS
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def collect_weworkremotely():
|
||||
"""Collect jobs from WeWorkRemotely RSS feed."""
|
||||
print("[4/5] Searching WeWorkRemotely RSS...")
|
||||
xml_text = fetch_text("https://weworkremotely.com/categories/remote-programming-jobs.rss")
|
||||
if not xml_text:
|
||||
return []
|
||||
|
||||
jobs = []
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
for item in root.findall(".//item"):
|
||||
title = item.findtext("title", "")
|
||||
desc = item.findtext("description", "")
|
||||
link = item.findtext("link", "")
|
||||
pub_date = item.findtext("pubDate", "")
|
||||
|
||||
full_text = f"{title} {strip_html(desc)}"
|
||||
if not is_claude_related(full_text):
|
||||
continue
|
||||
|
||||
# WWR title format: "Company: Job Title"
|
||||
company_match = re.match(r"^([^:]+):\s*(.+)", title)
|
||||
company = company_match.group(1).strip() if company_match else "Remote Company"
|
||||
position = company_match.group(2).strip() if company_match else title
|
||||
|
||||
# Convert RFC-2822 date to ISO 8601 for consistent sorting
|
||||
iso_date = pub_date
|
||||
if pub_date:
|
||||
try:
|
||||
from email.utils import parsedate_to_datetime
|
||||
dt = parsedate_to_datetime(pub_date)
|
||||
iso_date = dt.isoformat()
|
||||
except Exception:
|
||||
pass # Keep original if parsing fails
|
||||
|
||||
jobs.append({
|
||||
"id": f"wwr-{link.rstrip('/').split('/')[-1] if link else 'unknown'}",
|
||||
"company": company[:80],
|
||||
"position": position[:120],
|
||||
"location": "Remote",
|
||||
"remote": True,
|
||||
"salary": extract_salary(full_text),
|
||||
"description": truncate(strip_html(desc), 300),
|
||||
"applyUrl": link,
|
||||
"source": "WeWorkRemotely",
|
||||
"sourceUrl": link,
|
||||
"postedAt": iso_date,
|
||||
"tags": extract_tech_tags(full_text),
|
||||
"companyIcon": company_icon(company),
|
||||
})
|
||||
except ET.ParseError as e:
|
||||
print(f" [warn] XML parse error: {e}")
|
||||
|
||||
print(f" Found {len(jobs)} jobs from WeWorkRemotely")
|
||||
return jobs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source 5: Anthropic Careers (Greenhouse API)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def collect_anthropic_careers():
|
||||
"""Collect jobs from Anthropic's careers page via Greenhouse public API.
|
||||
|
||||
Strategy:
|
||||
1. Fetch all jobs (no content) — lightweight list of ~500+ positions
|
||||
2. Pre-filter by title keywords to find candidates (~50 max)
|
||||
3. Fetch full content for candidates individually
|
||||
4. Final filter for "Claude Code" mentions in description
|
||||
"""
|
||||
print("[5/5] Searching Anthropic Careers (Greenhouse API)...")
|
||||
data = fetch_json("https://boards-api.greenhouse.io/v1/boards/anthropic/jobs")
|
||||
if not data or "jobs" not in data:
|
||||
print(" [warn] Could not fetch Anthropic job listings")
|
||||
return []
|
||||
|
||||
all_listings = data["jobs"]
|
||||
print(f" Total Anthropic listings: {len(all_listings)}")
|
||||
|
||||
# Pre-filter: titles likely to mention Claude Code
|
||||
# Include broad engineering/product titles + anything with "claude" in title
|
||||
title_keywords = [
|
||||
"claude", "engineer", "developer", "architect", "platform",
|
||||
"infrastructure", "product", "design", "research", "ml ",
|
||||
"machine learning", "ai ", "applied", "full stack", "fullstack",
|
||||
"front end", "frontend", "back end", "backend", "devops", "sre",
|
||||
"security", "data", "sdk", "api", "tools", "dx", "evangelist",
|
||||
"communications", "technical", "software",
|
||||
]
|
||||
candidates = []
|
||||
for job in all_listings:
|
||||
title_lower = (job.get("title") or "").lower()
|
||||
if any(kw in title_lower for kw in title_keywords):
|
||||
candidates.append(job)
|
||||
|
||||
# No hard cap — title pre-filter already narrows the list sufficiently
|
||||
print(f" Pre-filtered to {len(candidates)} candidate roles, fetching content...")
|
||||
|
||||
jobs = []
|
||||
|
||||
def fetch_job_content(job_meta):
|
||||
"""Fetch full job details and check for Claude Code mentions."""
|
||||
job_id = job_meta.get("id")
|
||||
detail = fetch_json(
|
||||
f"https://boards-api.greenhouse.io/v1/boards/anthropic/jobs/{job_id}",
|
||||
timeout=15,
|
||||
)
|
||||
if not detail:
|
||||
return None
|
||||
|
||||
content_html = unescape(detail.get("content", "")) # Greenhouse double-escapes HTML
|
||||
title = detail.get("title", "")
|
||||
full_text = f"{title} {strip_html(content_html)}"
|
||||
|
||||
# Must specifically mention "Claude Code" (not just "Claude" — all Anthropic jobs mention Claude)
|
||||
claude_code_keywords = [
|
||||
"claude code", "claude-code", "claude coder",
|
||||
]
|
||||
if not any(kw in full_text.lower() for kw in claude_code_keywords):
|
||||
return None
|
||||
|
||||
# Parse location
|
||||
location_name = ""
|
||||
loc = detail.get("location", {})
|
||||
if isinstance(loc, dict):
|
||||
location_name = loc.get("name", "")
|
||||
|
||||
# Check remote from metadata
|
||||
remote = False
|
||||
for meta in detail.get("metadata", []):
|
||||
if meta.get("name", "").lower() in ("location type", "location_type"):
|
||||
val = (str(meta.get("value", "")) or "").lower()
|
||||
if "remote" in val:
|
||||
remote = True
|
||||
if not location_name:
|
||||
location_name = "Remote" if remote else "San Francisco, CA"
|
||||
elif remote and "remote" not in location_name.lower():
|
||||
location_name = f"{location_name} (Remote)"
|
||||
|
||||
# Salary from content
|
||||
salary = extract_salary(full_text)
|
||||
|
||||
# Department
|
||||
departments = detail.get("departments", [])
|
||||
dept_names = [d.get("name", "") for d in departments if d.get("name")]
|
||||
|
||||
# Tags
|
||||
tags = extract_tech_tags(full_text)
|
||||
if "Claude Code" not in tags:
|
||||
tags.insert(0, "Claude Code")
|
||||
if "Anthropic" not in tags:
|
||||
tags.append("Anthropic")
|
||||
|
||||
# Description — clean, skip boilerplate "About Anthropic" intro, and truncate
|
||||
clean_desc = strip_html(content_html)
|
||||
# Skip the generic "About Anthropic" intro paragraph — find the role-specific section
|
||||
for marker in ["The role", "The Role", "About the role", "About the Role",
|
||||
"What you'll do", "What You'll Do", "The position", "The Position",
|
||||
"We're looking", "We are looking", "This role", "In this role",
|
||||
"As a ", "As an ", "Join ", "You will ", "You'll "]:
|
||||
idx = clean_desc.find(marker)
|
||||
if idx > 0 and idx < 800:
|
||||
clean_desc = clean_desc[idx:]
|
||||
break
|
||||
# Strip "About the role" prefix itself if present
|
||||
clean_desc = re.sub(r"^About the [Rr]ole:?\s*", "", clean_desc)
|
||||
description = truncate(clean_desc, 300)
|
||||
|
||||
return {
|
||||
"id": f"anth-{job_id}",
|
||||
"company": "Anthropic",
|
||||
"position": title[:120],
|
||||
"location": location_name[:80],
|
||||
"remote": remote or "remote" in location_name.lower(),
|
||||
"salary": salary,
|
||||
"description": description,
|
||||
"applyUrl": detail.get("absolute_url", f"https://www.anthropic.com/careers/jobs"),
|
||||
"source": "Anthropic",
|
||||
"sourceUrl": detail.get("absolute_url", f"https://www.anthropic.com/careers/jobs"),
|
||||
"postedAt": detail.get("first_published", detail.get("updated_at", "")),
|
||||
"tags": tags,
|
||||
"companyIcon": "https://www.anthropic.com/favicon.ico",
|
||||
}
|
||||
|
||||
# Batch fetch with thread pool
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
futures = {executor.submit(fetch_job_content, c): c for c in candidates}
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
if result:
|
||||
jobs.append(result)
|
||||
|
||||
print(f" Found {len(jobs)} Claude Code jobs at Anthropic")
|
||||
return jobs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def generate():
|
||||
print("=" * 60)
|
||||
print("Claude Code Jobs Scraper (free sources only)")
|
||||
print("=" * 60)
|
||||
|
||||
all_jobs = []
|
||||
|
||||
# 1. HN Firebase (primary)
|
||||
hn_jobs = collect_hn_firebase()
|
||||
all_jobs.extend(hn_jobs)
|
||||
|
||||
# 2. HN Algolia (supplementary)
|
||||
existing_ids = {j["id"] for j in all_jobs}
|
||||
algolia_jobs = collect_hn_algolia(existing_ids)
|
||||
all_jobs.extend(algolia_jobs)
|
||||
|
||||
# 3. RemoteOK
|
||||
rok_jobs = collect_remoteok()
|
||||
all_jobs.extend(rok_jobs)
|
||||
|
||||
# 4. WeWorkRemotely
|
||||
wwr_jobs = collect_weworkremotely()
|
||||
all_jobs.extend(wwr_jobs)
|
||||
|
||||
# 5. Anthropic Careers
|
||||
anth_jobs = collect_anthropic_careers()
|
||||
all_jobs.extend(anth_jobs)
|
||||
|
||||
# Deduplicate by ID
|
||||
seen = set()
|
||||
unique = []
|
||||
for job in all_jobs:
|
||||
if job["id"] not in seen:
|
||||
seen.add(job["id"])
|
||||
unique.append(job)
|
||||
|
||||
# Sort by postedAt descending
|
||||
unique.sort(key=lambda j: j.get("postedAt", ""), reverse=True)
|
||||
|
||||
# Build output
|
||||
sources = sorted(set(j["source"] for j in unique))
|
||||
output = {
|
||||
"lastUpdated": datetime.now(timezone.utc).isoformat(),
|
||||
"totalJobs": len(unique),
|
||||
"sources": sources,
|
||||
"jobs": unique,
|
||||
}
|
||||
|
||||
# Write to docs/
|
||||
output_path = "docs/claude-jobs.json"
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(output, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Also copy to dashboard/public/ if it exists
|
||||
import os
|
||||
dashboard_path = "dashboard/public/claude-jobs.json"
|
||||
if os.path.isdir("dashboard/public"):
|
||||
with open(dashboard_path, "w", encoding="utf-8") as f:
|
||||
json.dump(output, f, indent=2, ensure_ascii=False)
|
||||
print(f"\nCopied to {dashboard_path}")
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 60)
|
||||
print(f"Total unique jobs: {len(unique)}")
|
||||
for src in sources:
|
||||
count = sum(1 for j in unique if j["source"] == src)
|
||||
print(f" {src}: {count}")
|
||||
remote_count = sum(1 for j in unique if j.get("remote"))
|
||||
print(f" Remote: {remote_count} / On-site: {len(unique) - remote_count}")
|
||||
print(f"\nSaved to {output_path}")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate()
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate Claude Code PRs data.
|
||||
|
||||
Fetches all PRs from the GitHub API that were created from branches
|
||||
starting with 'claude/' and writes a static JSON file for the dashboard.
|
||||
|
||||
Usage:
|
||||
python scripts/generate_claude_prs.py
|
||||
|
||||
Optionally set GITHUB_TOKEN env var to avoid rate limits.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import requests
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
REPO = "davila7/claude-code-templates"
|
||||
API_URL = f"https://api.github.com/repos/{REPO}/pulls"
|
||||
OUTPUT_FILE = os.path.join(os.path.dirname(__file__), "..", "docs", "claude-prs", "data.json")
|
||||
|
||||
|
||||
def fetch_prs(state="all", token=None):
|
||||
"""Fetch all PRs with pagination."""
|
||||
headers = {"Accept": "application/vnd.github+json"}
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
all_prs = []
|
||||
page = 1
|
||||
|
||||
while True:
|
||||
url = f"{API_URL}?state={state}&per_page=100&page={page}&sort=created&direction=desc"
|
||||
print(f" Fetching page {page}...")
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
resp = requests.get(url, headers=headers, timeout=30)
|
||||
break
|
||||
except requests.exceptions.RequestException as e:
|
||||
if attempt < 2:
|
||||
wait = 2 ** (attempt + 1)
|
||||
print(f" Retry in {wait}s: {e}")
|
||||
time.sleep(wait)
|
||||
else:
|
||||
raise
|
||||
|
||||
if resp.status_code == 403:
|
||||
print(f" Rate limited. Set GITHUB_TOKEN env var to increase limits.")
|
||||
break
|
||||
|
||||
if resp.status_code != 200:
|
||||
print(f" Error {resp.status_code}: {resp.text[:200]}")
|
||||
break
|
||||
|
||||
data = resp.json()
|
||||
if not data:
|
||||
break
|
||||
|
||||
all_prs.extend(data)
|
||||
page += 1
|
||||
|
||||
if len(data) < 100:
|
||||
break
|
||||
|
||||
return all_prs
|
||||
|
||||
|
||||
def is_claude_pr(pr):
|
||||
"""Check if a PR was created by Claude Code (branch starts with claude/)."""
|
||||
ref = pr.get("head", {}).get("ref", "")
|
||||
return ref.startswith("claude/")
|
||||
|
||||
|
||||
def extract_pr_data(pr):
|
||||
"""Extract only the fields needed for the dashboard."""
|
||||
return {
|
||||
"number": pr["number"],
|
||||
"title": pr["title"],
|
||||
"state": pr["state"],
|
||||
"merged": pr.get("merged_at") is not None,
|
||||
"branch": pr["head"]["ref"],
|
||||
"created_at": pr["created_at"],
|
||||
"merged_at": pr.get("merged_at"),
|
||||
"closed_at": pr.get("closed_at"),
|
||||
"url": pr["html_url"],
|
||||
"user": pr.get("user", {}).get("login", "unknown"),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
print("Generating Claude Code PRs data...")
|
||||
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
print(" Using GITHUB_TOKEN for authentication")
|
||||
else:
|
||||
print(" No GITHUB_TOKEN set (60 req/hr limit). Set it to avoid rate limits.")
|
||||
|
||||
all_prs = fetch_prs(token=token)
|
||||
print(f" Total PRs fetched: {len(all_prs)}")
|
||||
|
||||
claude_prs = [extract_pr_data(pr) for pr in all_prs if is_claude_pr(pr)]
|
||||
print(f" Claude Code PRs found: {len(claude_prs)}")
|
||||
|
||||
# Sort by creation date descending
|
||||
claude_prs.sort(key=lambda p: p["created_at"], reverse=True)
|
||||
|
||||
output = {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"repo": REPO,
|
||||
"total": len(claude_prs),
|
||||
"prs": claude_prs,
|
||||
}
|
||||
|
||||
os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
|
||||
with open(OUTPUT_FILE, "w") as f:
|
||||
json.dump(output, f, indent=2)
|
||||
|
||||
print(f" Written to {OUTPUT_FILE}")
|
||||
print("Done.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,966 @@
|
||||
import os
|
||||
import json
|
||||
import shutil
|
||||
import requests
|
||||
import subprocess
|
||||
from collections import defaultdict
|
||||
from dotenv import load_dotenv
|
||||
from pathlib import Path
|
||||
|
||||
# Load environment variables
|
||||
load_dotenv()
|
||||
|
||||
def run_security_validation():
|
||||
"""
|
||||
Run security validation on all components and return the results.
|
||||
Returns a dictionary with component paths as keys and security data as values.
|
||||
"""
|
||||
print("🔒 Running security validation on components...")
|
||||
|
||||
try:
|
||||
# Change to cli-tool directory to run npm command
|
||||
cli_tool_dir = Path(__file__).parent / 'cli-tool'
|
||||
|
||||
# Run security audit and generate JSON report
|
||||
result = subprocess.run(
|
||||
['npm', 'run', 'security-audit:json'],
|
||||
cwd=cli_tool_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300 # 5 minute timeout
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"⚠️ Security validation completed with warnings")
|
||||
print(f" stdout: {result.stdout[:200]}")
|
||||
print(f" stderr: {result.stderr[:200]}")
|
||||
else:
|
||||
print("✅ Security validation completed successfully")
|
||||
|
||||
# Read the generated security report
|
||||
security_report_path = cli_tool_dir / 'security-report.json'
|
||||
|
||||
if not security_report_path.exists():
|
||||
print("⚠️ Security report not found, skipping security metadata")
|
||||
return {}
|
||||
|
||||
with open(security_report_path, 'r', encoding='utf-8') as f:
|
||||
security_data = json.load(f)
|
||||
|
||||
# Transform security data into a lookup dictionary
|
||||
# Key format: "agents/category/name" -> security metadata
|
||||
security_lookup = {}
|
||||
|
||||
for component_result in security_data.get('components', []):
|
||||
component_info = component_result.get('component', {})
|
||||
component_path = component_info.get('path', '')
|
||||
component_type = component_info.get('type', '')
|
||||
|
||||
if component_path:
|
||||
# Extract category and name from path
|
||||
# Path format: components/agents/development-team/frontend-developer.md
|
||||
path_parts = component_path.replace('\\', '/').split('/')
|
||||
|
||||
# Handle both formats: "components/agents/..." and "cli-tool/components/agents/..."
|
||||
if 'components' in path_parts:
|
||||
components_idx = path_parts.index('components')
|
||||
if len(path_parts) > components_idx + 3:
|
||||
component_type = path_parts[components_idx + 1] # agents, commands, etc.
|
||||
category = path_parts[components_idx + 2]
|
||||
file_name = path_parts[components_idx + 3]
|
||||
name = os.path.splitext(file_name)[0]
|
||||
|
||||
# Create lookup key
|
||||
key = f"{component_type}/{category}/{name}"
|
||||
|
||||
# Extract security metadata
|
||||
overall = component_result.get('overall', {})
|
||||
validators = component_result.get('validators', {})
|
||||
|
||||
# Build validators object with detailed errors and warnings
|
||||
validators_data = {}
|
||||
for validator_name, validator_result in validators.items():
|
||||
# Process errors to extract line/column information
|
||||
processed_errors = []
|
||||
for error in validator_result.get('errors', []):
|
||||
error_data = {
|
||||
'level': error.get('level', 'error'),
|
||||
'code': error.get('code', ''),
|
||||
'message': error.get('message', ''),
|
||||
'timestamp': error.get('timestamp', '')
|
||||
}
|
||||
|
||||
# Extract metadata with line/column info
|
||||
metadata = error.get('metadata', {})
|
||||
if metadata:
|
||||
error_data['metadata'] = metadata
|
||||
|
||||
# Extract location info if available
|
||||
if 'line' in metadata:
|
||||
error_data['line'] = metadata['line']
|
||||
if 'column' in metadata:
|
||||
error_data['column'] = metadata['column']
|
||||
if 'position' in metadata:
|
||||
error_data['position'] = metadata['position']
|
||||
if 'lineText' in metadata:
|
||||
error_data['lineText'] = metadata['lineText']
|
||||
|
||||
# Extract examples array if present (for patterns with multiple matches)
|
||||
if 'examples' in metadata:
|
||||
error_data['examples'] = metadata['examples']
|
||||
|
||||
processed_errors.append(error_data)
|
||||
|
||||
# Process warnings similarly
|
||||
processed_warnings = []
|
||||
for warning in validator_result.get('warnings', []):
|
||||
warning_data = {
|
||||
'level': warning.get('level', 'warning'),
|
||||
'code': warning.get('code', ''),
|
||||
'message': warning.get('message', ''),
|
||||
'timestamp': warning.get('timestamp', '')
|
||||
}
|
||||
|
||||
# Extract metadata with line/column info
|
||||
metadata = warning.get('metadata', {})
|
||||
if metadata:
|
||||
warning_data['metadata'] = metadata
|
||||
|
||||
# Extract location info if available
|
||||
if 'line' in metadata:
|
||||
warning_data['line'] = metadata['line']
|
||||
if 'column' in metadata:
|
||||
warning_data['column'] = metadata['column']
|
||||
if 'position' in metadata:
|
||||
warning_data['position'] = metadata['position']
|
||||
if 'lineText' in metadata:
|
||||
warning_data['lineText'] = metadata['lineText']
|
||||
|
||||
# Extract examples array if present
|
||||
if 'examples' in metadata:
|
||||
warning_data['examples'] = metadata['examples']
|
||||
|
||||
processed_warnings.append(warning_data)
|
||||
|
||||
validators_data[validator_name] = {
|
||||
'valid': validator_result.get('valid', False),
|
||||
'score': validator_result.get('score', 0),
|
||||
'errorCount': validator_result.get('errorCount', 0),
|
||||
'warningCount': validator_result.get('warningCount', 0),
|
||||
'errors': processed_errors,
|
||||
'warnings': processed_warnings,
|
||||
'info': validator_result.get('info', [])
|
||||
}
|
||||
|
||||
security_lookup[key] = {
|
||||
'validated': True,
|
||||
'valid': overall.get('valid', False),
|
||||
'score': overall.get('score', 0),
|
||||
'errorCount': overall.get('errorCount', 0),
|
||||
'warningCount': overall.get('warningCount', 0),
|
||||
'lastValidated': security_data.get('timestamp', ''),
|
||||
'validators': validators_data
|
||||
}
|
||||
|
||||
# Add hash if available from integrity validator
|
||||
integrity_data = validators.get('integrity', {})
|
||||
if 'info' in integrity_data:
|
||||
for info_item in integrity_data['info']:
|
||||
# info_item is a dictionary with code, message, metadata
|
||||
if isinstance(info_item, dict):
|
||||
metadata = info_item.get('metadata', {})
|
||||
if 'fullHash' in metadata:
|
||||
security_lookup[key]['hash'] = metadata['fullHash']
|
||||
break
|
||||
elif 'hash' in metadata:
|
||||
security_lookup[key]['hash'] = metadata['hash']
|
||||
break
|
||||
|
||||
print(f"✅ Security metadata extracted for {len(security_lookup)} components")
|
||||
return security_lookup
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
print("⚠️ Security validation timed out after 5 minutes")
|
||||
return {}
|
||||
except FileNotFoundError:
|
||||
print("⚠️ npm command not found, skipping security validation")
|
||||
return {}
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error running security validation: {e}")
|
||||
return {}
|
||||
|
||||
def fetch_download_stats():
|
||||
"""
|
||||
Fetch download statistics from Supabase
|
||||
Returns a dictionary with component_type-component_name as key and download count as value
|
||||
Supports: agents, commands, mcps, settings, hooks, sandbox, skills, templates, plugins
|
||||
"""
|
||||
print("📊 Fetching download statistics from Supabase...")
|
||||
|
||||
# Define type mapping once (DRY principle)
|
||||
TYPE_MAPPING = {
|
||||
'agent': 'agents',
|
||||
'command': 'commands',
|
||||
'setting': 'settings',
|
||||
'hook': 'hooks',
|
||||
'mcp': 'mcps',
|
||||
'skill': 'skills',
|
||||
'template': 'templates',
|
||||
'plugin': 'plugins',
|
||||
'sandbox': 'sandbox'
|
||||
}
|
||||
|
||||
# Get Supabase credentials (same as generate_trending_data.py)
|
||||
supabase_url = os.getenv("SUPABASE_URL")
|
||||
supabase_api_key = os.getenv("SUPABASE_API_KEY")
|
||||
|
||||
if not supabase_url or not supabase_api_key:
|
||||
print("⚠️ Warning: Missing Supabase credentials, skipping download stats")
|
||||
return {}
|
||||
|
||||
try:
|
||||
headers = {
|
||||
'apikey': supabase_api_key,
|
||||
'Authorization': f'Bearer {supabase_api_key}'
|
||||
}
|
||||
|
||||
# First, let's query component_downloads to aggregate counts per component
|
||||
# We need to handle pagination since there can be many records
|
||||
api_url = f"{supabase_url}/rest/v1/component_downloads"
|
||||
|
||||
all_downloads = []
|
||||
offset = 0
|
||||
limit = 1000
|
||||
|
||||
# Fetch all records with pagination
|
||||
max_pages = 1000 # Safety limit (1000 pages * 1000 records = 1,000,000 max)
|
||||
for page in range(max_pages):
|
||||
paginated_headers = headers.copy()
|
||||
paginated_headers['Range'] = f'{offset}-{offset + limit - 1}'
|
||||
|
||||
response = requests.get(api_url, headers=paginated_headers)
|
||||
|
||||
if response.status_code not in [200, 206]:
|
||||
print(f" Page {page+1}: Got status {response.status_code}, stopping")
|
||||
break
|
||||
|
||||
batch = response.json()
|
||||
if not batch:
|
||||
break
|
||||
|
||||
all_downloads.extend(batch)
|
||||
|
||||
# Check if we have more records to fetch
|
||||
content_range = response.headers.get('content-range', '')
|
||||
|
||||
# If we get a range like "45000-45977/*", check if we got less than limit records
|
||||
if len(batch) < limit:
|
||||
break # We've reached the end
|
||||
|
||||
# Also check for explicit total if provided
|
||||
if content_range and '/' in content_range:
|
||||
parts = content_range.split('/')
|
||||
if parts[1] != '*':
|
||||
try:
|
||||
total = int(parts[1])
|
||||
if offset + limit >= total:
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
offset += limit
|
||||
|
||||
# Progress indicator every 10 pages
|
||||
if (page + 1) % 10 == 0:
|
||||
print(f" Fetched {len(all_downloads)} records so far...")
|
||||
|
||||
print(f"📊 Total records fetched: {len(all_downloads)}")
|
||||
|
||||
# If we fetched records, use them
|
||||
if len(all_downloads) > 0:
|
||||
# Process component_downloads data
|
||||
downloads = all_downloads
|
||||
|
||||
# Aggregate downloads by component
|
||||
download_counts = {}
|
||||
component_totals = defaultdict(int)
|
||||
|
||||
for download in downloads:
|
||||
component_type = download.get('component_type', '')
|
||||
component_name = download.get('component_name', '')
|
||||
|
||||
if component_type and component_name:
|
||||
# Handle case where component_name already includes category
|
||||
if '/' in component_name:
|
||||
category = component_name.split('/')[0]
|
||||
actual_name = component_name.split('/')[-1]
|
||||
else:
|
||||
actual_name = component_name
|
||||
category = 'general'
|
||||
|
||||
# Create a key for aggregation matching trending data structure
|
||||
key = f"{component_type}|{category}|{actual_name}"
|
||||
component_totals[key] += 1
|
||||
|
||||
# Convert to the format we need using the TYPE_MAPPING constant
|
||||
for key, count in component_totals.items():
|
||||
parts = key.split('|')
|
||||
if len(parts) == 3:
|
||||
component_type, category, component_name = parts
|
||||
mapped_type = TYPE_MAPPING.get(component_type, component_type + 's')
|
||||
final_key = f"{mapped_type}/{category}/{component_name}"
|
||||
download_counts[final_key] = count
|
||||
|
||||
print(f"✅ Fetched and aggregated {len(download_counts)} component download stats")
|
||||
return download_counts
|
||||
else:
|
||||
# Try alternative: fetch from download_stats table if it exists
|
||||
print("⚠️ No data from component_downloads, trying download_stats table...")
|
||||
alt_url = f"{supabase_url}/rest/v1/download_stats"
|
||||
alt_response = requests.get(alt_url, headers=headers)
|
||||
|
||||
if alt_response.status_code == 200:
|
||||
print("📊 Using download_stats table instead...")
|
||||
stats = alt_response.json()
|
||||
download_counts = {}
|
||||
|
||||
for stat in stats:
|
||||
component_type = stat.get('component_type', '')
|
||||
component_name = stat.get('component_name', '')
|
||||
total_downloads = stat.get('total_downloads', 0)
|
||||
|
||||
# Handle case where component_name already includes category
|
||||
if '/' in component_name:
|
||||
category = component_name.split('/')[0]
|
||||
actual_name = component_name.split('/')[-1]
|
||||
else:
|
||||
actual_name = component_name
|
||||
category = 'general'
|
||||
|
||||
# Map to plural form using TYPE_MAPPING constant
|
||||
mapped_type = TYPE_MAPPING.get(component_type, component_type + 's')
|
||||
key = f"{mapped_type}/{category}/{actual_name}"
|
||||
download_counts[key] = total_downloads
|
||||
|
||||
print(f"✅ Fetched stats for {len(download_counts)} components from download_stats")
|
||||
return download_counts
|
||||
else:
|
||||
print("⚠️ No download stats available")
|
||||
return {}
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error fetching download stats: {e}")
|
||||
return {}
|
||||
|
||||
def scan_directory_recursively(directory_path, relative_to_path=None):
|
||||
"""
|
||||
Recursively scan a directory and return all files with their relative paths.
|
||||
"""
|
||||
files_list = []
|
||||
|
||||
if not os.path.isdir(directory_path):
|
||||
return files_list
|
||||
|
||||
for root, dirs, files in os.walk(directory_path):
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
if relative_to_path:
|
||||
relative_path = os.path.relpath(file_path, relative_to_path)
|
||||
files_list.append(relative_path.replace("\\", "/"))
|
||||
else:
|
||||
files_list.append(file)
|
||||
|
||||
return files_list
|
||||
|
||||
def generate_components_json():
|
||||
"""
|
||||
Scans the cli-tool/components and cli-tool/templates directories and generates a components.json file
|
||||
for the static website, including the content of each file and download statistics.
|
||||
"""
|
||||
components_base_path = 'cli-tool/components'
|
||||
templates_base_path = 'cli-tool/templates'
|
||||
plugins_path = '.claude-plugin/marketplace.json'
|
||||
output_path = 'docs/components.json'
|
||||
components_data = {'agents': [], 'commands': [], 'mcps': [], 'settings': [], 'hooks': [], 'sandbox': [], 'skills': [], 'loops': [], 'templates': [], 'plugins': []}
|
||||
|
||||
# Run security validation
|
||||
security_metadata = run_security_validation()
|
||||
|
||||
# Fetch download statistics
|
||||
download_stats = fetch_download_stats()
|
||||
component_types = ['agents', 'commands', 'mcps', 'settings', 'hooks', 'sandbox', 'skills', 'loops']
|
||||
|
||||
print(f"Starting scan of {components_base_path} and {templates_base_path}...")
|
||||
|
||||
# Scan components (existing logic)
|
||||
for component_type in component_types:
|
||||
type_path = os.path.join(components_base_path, component_type)
|
||||
if not os.path.isdir(type_path):
|
||||
print(f"Warning: Directory not found for type: {component_type}")
|
||||
continue
|
||||
|
||||
print(f"Scanning for {component_type} in {type_path}...")
|
||||
|
||||
# Special handling for skills - they use SKILL.md inside category/skill directories
|
||||
if component_type == 'skills':
|
||||
for category in os.listdir(type_path):
|
||||
category_path = os.path.join(type_path, category)
|
||||
if os.path.isdir(category_path) and not category.endswith('.md'):
|
||||
for skill_dir in os.listdir(category_path):
|
||||
skill_dir_path = os.path.join(category_path, skill_dir)
|
||||
if os.path.isdir(skill_dir_path):
|
||||
# Look for SKILL.md inside the skill directory
|
||||
skill_file_path = os.path.join(skill_dir_path, 'SKILL.md')
|
||||
if os.path.isfile(skill_file_path):
|
||||
name = skill_dir # Use directory name as skill name
|
||||
|
||||
# Read file content
|
||||
content = ''
|
||||
description = ''
|
||||
try:
|
||||
with open(skill_file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Extract metadata from frontmatter if available
|
||||
author = ''
|
||||
repo = ''
|
||||
version = ''
|
||||
license_field = ''
|
||||
keywords = []
|
||||
if content.startswith('---'):
|
||||
frontmatter_end = content.find('---', 3)
|
||||
if frontmatter_end != -1:
|
||||
frontmatter = content[3:frontmatter_end]
|
||||
for line in frontmatter.split('\n'):
|
||||
if line.startswith('description:'):
|
||||
description = line.split('description:', 1)[1].strip()
|
||||
elif line.startswith('author:'):
|
||||
author = line.split('author:', 1)[1].strip()
|
||||
elif line.startswith('repo:'):
|
||||
repo = line.split('repo:', 1)[1].strip()
|
||||
elif line.startswith('version:'):
|
||||
version = line.split('version:', 1)[1].strip()
|
||||
elif line.startswith('license:'):
|
||||
license_field = line.split('license:', 1)[1].strip()
|
||||
elif line.startswith('tags:'):
|
||||
tags_str = line.split('tags:', 1)[1].strip()
|
||||
# Parse tags array [tag1, tag2, tag3]
|
||||
if tags_str.startswith('[') and tags_str.endswith(']'):
|
||||
tags_str = tags_str[1:-1]
|
||||
keywords = [tag.strip() for tag in tags_str.split(',')]
|
||||
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read file {skill_file_path}: {e}")
|
||||
|
||||
# Look up download count for this skill
|
||||
download_key = f"{component_type}/{category}/{name}"
|
||||
downloads = download_stats.get(download_key, 0)
|
||||
|
||||
# Look up security metadata for this skill
|
||||
security_key = f"{component_type}/{category}/{name}"
|
||||
security = security_metadata.get(security_key, {
|
||||
'validated': False,
|
||||
'valid': None,
|
||||
'score': None,
|
||||
'errorCount': 0,
|
||||
'warningCount': 0,
|
||||
'lastValidated': None
|
||||
})
|
||||
|
||||
# Collect reference files (all files except SKILL.md)
|
||||
references = []
|
||||
for ref_file in scan_directory_recursively(skill_dir_path, skill_dir_path):
|
||||
if ref_file != 'SKILL.md':
|
||||
references.append(ref_file)
|
||||
references.sort()
|
||||
|
||||
# Path includes category for proper organization
|
||||
component = {
|
||||
'name': name, # Just the skill directory name
|
||||
'path': f"{category}/{name}", # category/skill-name format
|
||||
'category': category,
|
||||
'type': 'skill',
|
||||
'content': content,
|
||||
'description': description,
|
||||
'author': author,
|
||||
'repo': repo,
|
||||
'version': version,
|
||||
'license': license_field,
|
||||
'keywords': keywords,
|
||||
'downloads': downloads,
|
||||
'security': security,
|
||||
'references': references
|
||||
}
|
||||
components_data[component_type].append(component)
|
||||
print(f" Processed skill: {category}/{name}")
|
||||
continue # Skip the normal file scanning for skills
|
||||
|
||||
# Normal scanning for other component types
|
||||
for category in os.listdir(type_path):
|
||||
category_path = os.path.join(type_path, category)
|
||||
if os.path.isdir(category_path):
|
||||
for file_name in os.listdir(category_path):
|
||||
file_path = os.path.join(category_path, file_name)
|
||||
if os.path.isfile(file_path) and (file_name.endswith('.md') or file_name.endswith('.json')) and not file_name.endswith('.py'):
|
||||
name = os.path.splitext(file_name)[0]
|
||||
|
||||
# Read file content
|
||||
content = ''
|
||||
description = ''
|
||||
author = ''
|
||||
repo = ''
|
||||
version = ''
|
||||
license_field = ''
|
||||
keywords = []
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Extract description, author and repo from JSON files
|
||||
if file_name.endswith('.json'):
|
||||
try:
|
||||
import json
|
||||
json_data = json.loads(content)
|
||||
|
||||
if component_type == 'mcps':
|
||||
# Extract description from the first mcpServer entry
|
||||
if 'mcpServers' in json_data:
|
||||
for server_name, server_config in json_data['mcpServers'].items():
|
||||
if isinstance(server_config, dict) and 'description' in server_config:
|
||||
description = server_config['description']
|
||||
break # Use the first description found
|
||||
elif component_type in ['settings', 'hooks']:
|
||||
# Extract metadata from settings/hooks JSON files
|
||||
if 'description' in json_data:
|
||||
description = json_data['description']
|
||||
if 'author' in json_data:
|
||||
author = json_data['author']
|
||||
if 'repo' in json_data:
|
||||
repo = json_data['repo']
|
||||
if 'version' in json_data:
|
||||
version = json_data['version']
|
||||
if 'license' in json_data:
|
||||
license_field = json_data['license']
|
||||
if 'keywords' in json_data and isinstance(json_data['keywords'], list):
|
||||
keywords = json_data['keywords']
|
||||
|
||||
except json.JSONDecodeError:
|
||||
print(f"Warning: Invalid JSON in {file_path}")
|
||||
|
||||
# Extract metadata from markdown frontmatter
|
||||
elif file_name.endswith('.md'):
|
||||
if content.startswith('---'):
|
||||
frontmatter_end = content.find('---', 3)
|
||||
if frontmatter_end != -1:
|
||||
frontmatter = content[3:frontmatter_end]
|
||||
for line in frontmatter.split('\n'):
|
||||
if line.startswith('description:'):
|
||||
description = line.split('description:', 1)[1].strip()
|
||||
elif line.startswith('author:'):
|
||||
author = line.split('author:', 1)[1].strip()
|
||||
elif line.startswith('repo:'):
|
||||
repo = line.split('repo:', 1)[1].strip()
|
||||
elif line.startswith('version:'):
|
||||
version = line.split('version:', 1)[1].strip()
|
||||
elif line.startswith('license:'):
|
||||
license_field = line.split('license:', 1)[1].strip()
|
||||
elif line.startswith('tags:'):
|
||||
tags_str = line.split('tags:', 1)[1].strip()
|
||||
# Parse tags array [tag1, tag2, tag3]
|
||||
if tags_str.startswith('[') and tags_str.endswith(']'):
|
||||
tags_str = tags_str[1:-1]
|
||||
keywords = [tag.strip() for tag in tags_str.split(',')]
|
||||
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read file {file_path}: {e}")
|
||||
|
||||
# Look up download count for this component
|
||||
# The key in download_stats uses the plural form: agents/category/name
|
||||
download_key = f"{component_type}/{category}/{name}"
|
||||
downloads = download_stats.get(download_key, 0)
|
||||
|
||||
# Look up security metadata for this component
|
||||
security_key = f"{component_type}/{category}/{name}"
|
||||
security = security_metadata.get(security_key, {
|
||||
'validated': False,
|
||||
'valid': None,
|
||||
'score': None,
|
||||
'errorCount': 0,
|
||||
'warningCount': 0,
|
||||
'lastValidated': None
|
||||
})
|
||||
|
||||
component = {
|
||||
'name': name,
|
||||
'path': os.path.join(category, file_name).replace("\\", "/"),
|
||||
'category': category,
|
||||
'type': component_type[:-1], # singular form
|
||||
'content': content, # Add file content
|
||||
'description': description, # Add description for MCPs
|
||||
'author': author, # Add author field
|
||||
'repo': repo, # Add repository field
|
||||
'version': version, # Add version field
|
||||
'license': license_field, # Add license field
|
||||
'keywords': keywords, # Add keywords field
|
||||
'downloads': downloads, # Add download count
|
||||
'security': security # Add security metadata
|
||||
}
|
||||
components_data[component_type].append(component)
|
||||
|
||||
# Scan templates (new logic)
|
||||
if os.path.isdir(templates_base_path):
|
||||
print(f"Scanning for templates in {templates_base_path}...")
|
||||
|
||||
for language_dir in os.listdir(templates_base_path):
|
||||
language_path = os.path.join(templates_base_path, language_dir)
|
||||
if os.path.isdir(language_path):
|
||||
print(f" Processing language: {language_dir}")
|
||||
|
||||
# Collect files in the language directory (excluding examples folder)
|
||||
language_files = []
|
||||
for item in os.listdir(language_path):
|
||||
item_path = os.path.join(language_path, item)
|
||||
if os.path.isfile(item_path):
|
||||
language_files.append(item)
|
||||
elif os.path.isdir(item_path) and item != 'examples':
|
||||
# For directories like .claude, scan recursively
|
||||
if item == '.claude':
|
||||
recursive_files = scan_directory_recursively(item_path, language_path)
|
||||
language_files.extend(recursive_files)
|
||||
else:
|
||||
# For other directories, just add the directory name (optional)
|
||||
pass
|
||||
|
||||
# Look up download count for this template
|
||||
template_download_key = f"templates/{language_dir}"
|
||||
template_downloads = download_stats.get(template_download_key, 0)
|
||||
|
||||
# Create language template entry
|
||||
language_template = {
|
||||
'name': language_dir,
|
||||
'id': language_dir,
|
||||
'type': 'template',
|
||||
'subtype': 'language',
|
||||
'category': 'languages',
|
||||
'description': f'{language_dir.title()} project template',
|
||||
'files': language_files,
|
||||
'installCommand': f'npx claude-code-templates@latest --template={language_dir} --yes',
|
||||
'downloads': template_downloads
|
||||
}
|
||||
components_data['templates'].append(language_template)
|
||||
|
||||
# Check for examples folder (contains frameworks)
|
||||
examples_path = os.path.join(language_path, 'examples')
|
||||
if os.path.isdir(examples_path):
|
||||
for framework_dir in os.listdir(examples_path):
|
||||
framework_path = os.path.join(examples_path, framework_dir)
|
||||
if os.path.isdir(framework_path):
|
||||
print(f" Processing framework: {framework_dir}")
|
||||
|
||||
# Collect files in the framework directory
|
||||
framework_files = []
|
||||
for item in os.listdir(framework_path):
|
||||
item_path = os.path.join(framework_path, item)
|
||||
if os.path.isfile(item_path):
|
||||
framework_files.append(item)
|
||||
elif os.path.isdir(item_path):
|
||||
# For directories like .claude, scan recursively
|
||||
if item == '.claude':
|
||||
recursive_files = scan_directory_recursively(item_path, framework_path)
|
||||
framework_files.extend(recursive_files)
|
||||
else:
|
||||
# For other directories, just add the directory name (optional)
|
||||
pass
|
||||
|
||||
# Look up download count for this framework
|
||||
framework_download_key = f"templates/{framework_dir}"
|
||||
framework_downloads = download_stats.get(framework_download_key, 0)
|
||||
|
||||
# Create framework template entry
|
||||
framework_template = {
|
||||
'name': framework_dir,
|
||||
'id': framework_dir,
|
||||
'type': 'template',
|
||||
'subtype': 'framework',
|
||||
'category': 'frameworks',
|
||||
'language': language_dir,
|
||||
'description': f'{framework_dir.title()} with {language_dir.title()}',
|
||||
'files': framework_files,
|
||||
'installCommand': f'npx claude-code-templates@latest --template={framework_dir} --yes',
|
||||
'downloads': framework_downloads
|
||||
}
|
||||
components_data['templates'].append(framework_template)
|
||||
else:
|
||||
print(f"Warning: Templates directory not found: {templates_base_path}")
|
||||
|
||||
# Load components metadata from marketplace.json (Claude Code standard)
|
||||
components_marketplace_path = 'cli-tool/components/.claude-plugin/marketplace.json'
|
||||
components_marketplace = None
|
||||
if os.path.isfile(components_marketplace_path):
|
||||
print(f"Loading components metadata from {components_marketplace_path}...")
|
||||
try:
|
||||
with open(components_marketplace_path, 'r', encoding='utf-8') as f:
|
||||
components_marketplace = json.load(f)
|
||||
print(f"✅ Loaded metadata for {len(components_marketplace.get('agents', []))} agents")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error reading components metadata from {components_marketplace_path}: {e}")
|
||||
|
||||
# Scan plugins from marketplace.json and store marketplace data
|
||||
marketplace_full_data = None
|
||||
if os.path.isfile(plugins_path):
|
||||
print(f"Scanning for plugins in {plugins_path}...")
|
||||
try:
|
||||
with open(plugins_path, 'r', encoding='utf-8') as f:
|
||||
marketplace_data = json.load(f)
|
||||
|
||||
# Store full marketplace data for inclusion in components.json
|
||||
marketplace_full_data = marketplace_data
|
||||
|
||||
if 'plugins' in marketplace_data:
|
||||
for plugin in marketplace_data['plugins']:
|
||||
plugin_name = plugin.get('name', '')
|
||||
plugin_description = plugin.get('description', '')
|
||||
plugin_version = plugin.get('version', '1.0.0')
|
||||
plugin_keywords = plugin.get('keywords', [])
|
||||
plugin_author = plugin.get('author', {}).get('name', '')
|
||||
|
||||
# Get component lists with file names
|
||||
commands_list = plugin.get('commands', [])
|
||||
agents_list = plugin.get('agents', [])
|
||||
mcps_list = plugin.get('mcpServers', [])
|
||||
|
||||
# Extract component names with category/folder from file paths
|
||||
def extract_component_with_category(path):
|
||||
# Extract category/name from path like "./cli-tool/components/commands/git/feature.md"
|
||||
# Result should be "git/feature"
|
||||
parts = path.split('/')
|
||||
if len(parts) >= 2:
|
||||
filename = parts[-1].replace('.md', '').replace('.json', '')
|
||||
category = parts[-2]
|
||||
return f"{category}/{filename}"
|
||||
else:
|
||||
# Fallback to just filename if path structure is unexpected
|
||||
filename = path.split('/')[-1]
|
||||
return filename.replace('.md', '').replace('.json', '')
|
||||
|
||||
commands_names = [extract_component_with_category(cmd) for cmd in commands_list]
|
||||
agents_names = [extract_component_with_category(agent) for agent in agents_list]
|
||||
mcps_names = [extract_component_with_category(mcp) for mcp in mcps_list]
|
||||
|
||||
# Look up download count for this plugin
|
||||
plugin_download_key = f"plugins/{plugin_name}"
|
||||
plugin_downloads = download_stats.get(plugin_download_key, 0)
|
||||
|
||||
# Create plugin entry
|
||||
plugin_entry = {
|
||||
'name': plugin_name,
|
||||
'id': plugin_name,
|
||||
'type': 'plugin',
|
||||
'description': plugin_description,
|
||||
'version': plugin_version,
|
||||
'keywords': plugin_keywords,
|
||||
'author': plugin_author,
|
||||
'commands': len(commands_list),
|
||||
'agents': len(agents_list),
|
||||
'mcpServers': len(mcps_list),
|
||||
'commandsList': commands_names,
|
||||
'agentsList': agents_names,
|
||||
'mcpServersList': mcps_names,
|
||||
'installCommand': f'/plugin install {plugin_name}@claude-code-templates',
|
||||
'downloads': plugin_downloads
|
||||
}
|
||||
components_data['plugins'].append(plugin_entry)
|
||||
print(f" Processed plugin: {plugin_name}")
|
||||
|
||||
print(f"✅ Processed {len(components_data['plugins'])} plugins from marketplace.json")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error reading plugins from {plugins_path}: {e}")
|
||||
else:
|
||||
print(f"Warning: Plugins file not found: {plugins_path}")
|
||||
|
||||
# Sort components alphabetically
|
||||
for component_type in components_data:
|
||||
if component_type in ['templates', 'plugins']:
|
||||
# Sort templates and plugins by name since they don't have path
|
||||
components_data[component_type].sort(key=lambda x: x['name'])
|
||||
else:
|
||||
# Sort other components by path
|
||||
components_data[component_type].sort(key=lambda x: x['path'])
|
||||
|
||||
# Add marketplace metadata (root level - our public marketplace)
|
||||
if marketplace_full_data:
|
||||
components_data['marketplace'] = marketplace_full_data
|
||||
print("✅ Added public marketplace metadata to components.json")
|
||||
|
||||
# Add components marketplace metadata (Claude Code standard)
|
||||
if components_marketplace:
|
||||
components_data['componentsMarketplace'] = components_marketplace
|
||||
print("✅ Added components marketplace metadata to components.json")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Performance optimization: separate content from the lightweight index.
|
||||
#
|
||||
# docs/components.json → index without 'content' (~87% smaller)
|
||||
# dashboard/public/components.json → same, served by Cloudflare Pages
|
||||
# dashboard/public/component-content/{type}/{slug}.json → per-component
|
||||
# raw content, loaded on demand only on detail pages.
|
||||
# ---------------------------------------------------------------------------
|
||||
dashboard_public_dir = 'dashboard/public'
|
||||
dashboard_content_dir = os.path.join(dashboard_public_dir, 'component-content')
|
||||
|
||||
# Component types that carry a 'content' field
|
||||
content_bearing_types = ['agents', 'commands', 'mcps', 'settings', 'hooks', 'sandbox', 'skills', 'loops']
|
||||
|
||||
# 1. Write per-component content files directly to dashboard/public/component-content/
|
||||
os.makedirs(dashboard_content_dir, exist_ok=True)
|
||||
written_content_count = 0
|
||||
for ctype in content_bearing_types:
|
||||
for component in components_data.get(ctype, []):
|
||||
raw_content = component.get('content', '')
|
||||
if not raw_content:
|
||||
continue
|
||||
# Slug: path with extension stripped (matches detail-page URL lookup)
|
||||
slug = component['path'].replace('.md', '').replace('.json', '')
|
||||
content_file_path = os.path.join(dashboard_content_dir, ctype, slug + '.json')
|
||||
os.makedirs(os.path.dirname(content_file_path), exist_ok=True)
|
||||
try:
|
||||
with open(content_file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump({'content': raw_content}, f, ensure_ascii=False)
|
||||
written_content_count += 1
|
||||
except IOError as e:
|
||||
print(f"Warning: Could not write content file {content_file_path}: {e}")
|
||||
|
||||
print(f"Generated {written_content_count} component content files in {dashboard_content_dir}/")
|
||||
|
||||
# 2. Build the legacy/docs index: strip only 'content'.
|
||||
# The legacy static site (docs/js/*) renders security badges, so the
|
||||
# 'security' field MUST stay in docs/components.json.
|
||||
index_data = {}
|
||||
for k, v in components_data.items():
|
||||
if k in content_bearing_types:
|
||||
index_data[k] = [{key: val for key, val in c.items() if key != 'content'} for c in v]
|
||||
else:
|
||||
index_data[k] = v
|
||||
|
||||
# 3. Write the docs index (with 'security', without 'content').
|
||||
try:
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(index_data, f, indent=2, ensure_ascii=False)
|
||||
print(f"Successfully generated {output_path} (index without content).")
|
||||
except IOError as e:
|
||||
print(f"Error writing to {output_path}: {e}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dashboard-specific artifacts (Cloudflare Pages, www.aitmpl.com).
|
||||
#
|
||||
# The dashboard never reads the 'security' field, so we strip it here to
|
||||
# shrink the payload the browser downloads. We also emit smaller,
|
||||
# purpose-built artifacts so islands stop downloading the whole index:
|
||||
#
|
||||
# components.json → full index, no 'content', no 'security'
|
||||
# counts.json → { type: count } (sidebars/plugins pages)
|
||||
# components/{type}.json → per-type slice (grid loads only active type)
|
||||
# search-index.json → flat [{type,name,path,description,category}]
|
||||
# ---------------------------------------------------------------------------
|
||||
STRIP_KEYS = {'content', 'security'}
|
||||
|
||||
def strip_for_dashboard(component):
|
||||
return {key: val for key, val in component.items() if key not in STRIP_KEYS}
|
||||
|
||||
# Build the dashboard index (content + security stripped from components).
|
||||
dashboard_index_data = {}
|
||||
for k, v in index_data.items():
|
||||
if k in content_bearing_types and isinstance(v, list):
|
||||
dashboard_index_data[k] = [strip_for_dashboard(c) for c in v]
|
||||
elif isinstance(v, list):
|
||||
# Non-content types (e.g. mcps/settings/hooks may carry security)
|
||||
dashboard_index_data[k] = [strip_for_dashboard(c) for c in v]
|
||||
else:
|
||||
dashboard_index_data[k] = v
|
||||
|
||||
try:
|
||||
os.makedirs(dashboard_public_dir, exist_ok=True)
|
||||
dashboard_index_path = os.path.join(dashboard_public_dir, 'components.json')
|
||||
with open(dashboard_index_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(dashboard_index_data, f, ensure_ascii=False)
|
||||
print(f"Generated {dashboard_index_path} (dashboard index, no content/security).")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not write dashboard index: {e}")
|
||||
|
||||
# counts.json — tiny per-type counts so sidebars/plugins pages stop
|
||||
# downloading the full index just to read array lengths.
|
||||
try:
|
||||
counts = {k: len(v) for k, v in dashboard_index_data.items() if isinstance(v, list)}
|
||||
counts_path = os.path.join(dashboard_public_dir, 'counts.json')
|
||||
with open(counts_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(counts, f, ensure_ascii=False)
|
||||
print(f"Generated {counts_path} ({counts}).")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not write counts.json: {e}")
|
||||
|
||||
# components/{type}.json — per-type slices so the grid loads only the
|
||||
# active type instead of all 1800+ components at once.
|
||||
try:
|
||||
per_type_dir = os.path.join(dashboard_public_dir, 'components')
|
||||
os.makedirs(per_type_dir, exist_ok=True)
|
||||
for k, v in dashboard_index_data.items():
|
||||
if not isinstance(v, list):
|
||||
continue
|
||||
type_path = os.path.join(per_type_dir, f'{k}.json')
|
||||
with open(type_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(v, f, ensure_ascii=False)
|
||||
print(f"Generated per-type index files in {per_type_dir}/")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not write per-type index files: {e}")
|
||||
|
||||
# search-index.json — flat array with only the fields the global (Cmd+K)
|
||||
# search reads: type, name, path, description, category.
|
||||
try:
|
||||
search_index = []
|
||||
for k, v in dashboard_index_data.items():
|
||||
if not isinstance(v, list):
|
||||
continue
|
||||
for c in v:
|
||||
search_index.append({
|
||||
'type': c.get('type'),
|
||||
'name': c.get('name'),
|
||||
'path': c.get('path'),
|
||||
'description': c.get('description', ''),
|
||||
'category': c.get('category'),
|
||||
})
|
||||
search_path = os.path.join(dashboard_public_dir, 'search-index.json')
|
||||
with open(search_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(search_index, f, ensure_ascii=False)
|
||||
print(f"Generated {search_path} ({len(search_index)} entries).")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not write search-index.json: {e}")
|
||||
|
||||
# Log summary (uses original components_data which still has content for accurate counts)
|
||||
print("\n--- Generation Summary ---")
|
||||
for component_type, components in components_data.items():
|
||||
# Skip marketplace metadata in summary (it's not a component type)
|
||||
if component_type in ['marketplace', 'componentsMarketplace']:
|
||||
continue
|
||||
|
||||
print(f" - Found and processed {len(components)} {component_type}")
|
||||
if component_type == 'templates':
|
||||
languages = len([t for t in components if t.get('subtype') == 'language'])
|
||||
frameworks = len([t for t in components if t.get('subtype') == 'framework'])
|
||||
print(f" • {languages} languages, {frameworks} frameworks")
|
||||
elif component_type == 'plugins':
|
||||
total_commands = sum(p.get('commands', 0) for p in components)
|
||||
total_agents = sum(p.get('agents', 0) for p in components)
|
||||
total_mcps = sum(p.get('mcpServers', 0) for p in components)
|
||||
print(f" • {total_commands} commands, {total_agents} agents, {total_mcps} MCPs")
|
||||
|
||||
# Security statistics for components with security metadata
|
||||
if component_type not in ['templates', 'plugins']:
|
||||
validated = len([c for c in components if c.get('security', {}).get('validated', False)])
|
||||
valid_components = len([c for c in components if c.get('security', {}).get('valid', False)])
|
||||
avg_score = sum(c.get('security', {}).get('score', 0) for c in components if c.get('security', {}).get('validated', False)) / validated if validated > 0 else 0
|
||||
print(f" • Security: {validated} validated, {valid_components} passed, avg score: {avg_score:.1f}")
|
||||
|
||||
print("--------------------------")
|
||||
|
||||
if __name__ == '__main__':
|
||||
generate_components_json()
|
||||
@@ -0,0 +1,660 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Plugins JSON Generator Script
|
||||
Fetches .claude-plugin/ data from GitHub repos and generates plugins.json
|
||||
with real marketplace.json and plugin.json contents.
|
||||
|
||||
Usage:
|
||||
python scripts/generate_plugins_json.py
|
||||
|
||||
Requires:
|
||||
- GITHUB_TOKEN env var (or gh CLI authenticated)
|
||||
- pip install requests python-dotenv
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import subprocess
|
||||
|
||||
# Build a clean env for gh CLI (avoid conflicting tokens)
|
||||
_GH_ENV = {k: v for k, v in os.environ.items()}
|
||||
# Remove any stale GitHub tokens that might override gh CLI auth
|
||||
for _key in ("GITHUB_TOKEN", "GH_TOKEN", "GITHUB_ENTERPRISE_TOKEN"):
|
||||
_GH_ENV.pop(_key, None)
|
||||
|
||||
# --- Configuration ---
|
||||
|
||||
# Repos to scan: (owner/repo, optional website URL)
|
||||
REPOS = [
|
||||
("anthropics/claude-plugins-official", None),
|
||||
("anthropics/knowledge-work-plugins", None),
|
||||
("affaan-m/everything-claude-code", None),
|
||||
("thedotmack/claude-mem", None),
|
||||
("jarrodwatts/claude-hud", None),
|
||||
("EveryInc/compound-engineering-plugin", None),
|
||||
("alirezarezvani/claude-skills", None),
|
||||
("davepoon/buildwithclaude", "https://buildwithclaude.com/"),
|
||||
("Nyanbalaji28/best-claude-skills", "https://augmentclaude.com/"),
|
||||
("lackeyjb/playwright-skill", None),
|
||||
("nyldn/claude-octopus", None),
|
||||
("jeremylongshore/claude-code-plugins-plus-skills", None),
|
||||
("timescale/pg-aiguide", None),
|
||||
("CloudAI-X/claude-workflow-v2", None),
|
||||
("numman-ali/n-skills", None),
|
||||
("obra/superpowers-marketplace", None),
|
||||
("muratcankoylan/ralph-wiggum-marketer", None),
|
||||
("team-attention/plugins-for-claude-natives", None),
|
||||
("ananddtyagi/cc-marketplace", None),
|
||||
("agent-sh/agentsys", None),
|
||||
("ccplugins/awesome-claude-code-plugins", None),
|
||||
("hamelsmu/claude-review-loop", None),
|
||||
("sangrokjung/claude-forge", None),
|
||||
("gmickel/gmickel-claude-marketplace", None),
|
||||
("kingbootoshi/cartographer", None),
|
||||
("zscole/adversarial-spec", None),
|
||||
("hashicorp/agent-skills", None),
|
||||
("quant-sentiment-ai/claude-equity-research", None),
|
||||
("777genius/claude-notifications-go", None),
|
||||
("Piebald-AI/claude-code-lsps", None),
|
||||
("chu2bard/pinion-os", None),
|
||||
("Airtable/skills", None),
|
||||
]
|
||||
|
||||
OUTPUT_PATH = os.path.join(os.path.dirname(__file__), "..", "dashboard", "public", "plugins.json")
|
||||
|
||||
# --- GitHub API helpers (uses gh CLI) ---
|
||||
|
||||
_request_count = 0
|
||||
|
||||
|
||||
def gh_api(endpoint, retries=3):
|
||||
"""Call GitHub API via gh CLI subprocess."""
|
||||
global _request_count
|
||||
|
||||
for attempt in range(retries):
|
||||
_request_count += 1
|
||||
# Respect rate limits
|
||||
if _request_count % 50 == 0:
|
||||
time.sleep(1)
|
||||
|
||||
try:
|
||||
cmd = ["/usr/local/bin/gh", "api", endpoint.lstrip("/")]
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, text=True, timeout=30, env=_GH_ENV
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
return json.loads(result.stdout)
|
||||
if "Not Found" in result.stderr or "404" in result.stderr:
|
||||
return None
|
||||
if "rate limit" in result.stderr.lower():
|
||||
wait = 30 * (attempt + 1)
|
||||
print(f" ⏳ Rate limited. Waiting {wait}s...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
if result.returncode != 0:
|
||||
if attempt < retries - 1:
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
return None
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError) as e:
|
||||
if attempt < retries - 1:
|
||||
time.sleep(2 ** attempt)
|
||||
else:
|
||||
print(f" ⚠️ gh CLI failed: {e}")
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def gh_file_content(repo, path):
|
||||
"""Fetch and decode a file from a GitHub repo using gh CLI."""
|
||||
global _request_count
|
||||
_request_count += 1
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["/usr/local/bin/gh", "api", f"repos/{repo}/contents/{path}", "--jq", ".content"],
|
||||
capture_output=True, text=True, timeout=30, env=_GH_ENV
|
||||
)
|
||||
if result.returncode != 0 or not result.stdout.strip():
|
||||
return None
|
||||
import base64
|
||||
return base64.b64decode(result.stdout.strip()).decode("utf-8")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def gh_dir_listing(repo, path):
|
||||
"""List files/dirs at a path in a GitHub repo."""
|
||||
data = gh_api(f"repos/{repo}/contents/{path}")
|
||||
if not data or not isinstance(data, list):
|
||||
return []
|
||||
return data
|
||||
|
||||
|
||||
# --- Plugin data extraction ---
|
||||
|
||||
def parse_json_safe(text):
|
||||
"""Parse JSON, tolerating trailing commas and minor issues."""
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
# Try stripping BOM or trailing garbage
|
||||
text = text.strip().lstrip("\ufeff")
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def extract_plugin_components(plugin_json):
|
||||
"""Extract component counts from a plugin.json manifest."""
|
||||
if not plugin_json:
|
||||
return {}
|
||||
counts = {}
|
||||
for key in ("skills", "agents", "commands"):
|
||||
val = plugin_json.get(key)
|
||||
if isinstance(val, list):
|
||||
# Count only actual file entries, not directory paths
|
||||
file_entries = [v for v in val if isinstance(v, str) and not v.endswith("/")]
|
||||
dir_entries = [v for v in val if isinstance(v, str) and v.endswith("/")]
|
||||
# File entries count directly; dir entries count as 1 (minimum)
|
||||
count = len(file_entries) + len(dir_entries)
|
||||
if count > 0:
|
||||
counts[key] = count
|
||||
elif isinstance(val, str):
|
||||
counts[key] = 1
|
||||
# Hooks
|
||||
hooks = plugin_json.get("hooks")
|
||||
if hooks:
|
||||
if isinstance(hooks, list):
|
||||
counts["hooks"] = len(hooks)
|
||||
elif isinstance(hooks, str):
|
||||
counts["hooks"] = 1
|
||||
# MCPs
|
||||
mcps = plugin_json.get("mcpServers")
|
||||
if isinstance(mcps, dict) and len(mcps) > 0:
|
||||
counts["mcps"] = len(mcps)
|
||||
# LSPs
|
||||
lsps = plugin_json.get("lspServers")
|
||||
if isinstance(lsps, dict) and len(lsps) > 0:
|
||||
counts["lsps"] = len(lsps)
|
||||
# Rules
|
||||
rules = plugin_json.get("rules")
|
||||
if rules:
|
||||
counts["rules"] = 1 if isinstance(rules, str) else len(rules)
|
||||
return counts
|
||||
|
||||
|
||||
def classify_repo_type(marketplace, plugin_json):
|
||||
"""Determine if repo is a marketplace or single plugin."""
|
||||
if not marketplace:
|
||||
return "plugin"
|
||||
plugins_list = marketplace.get("plugins", [])
|
||||
if len(plugins_list) > 1:
|
||||
return "marketplace"
|
||||
# Single plugin in marketplace = it's a plugin repo
|
||||
return "plugin"
|
||||
|
||||
|
||||
def extract_tags_from_components(components):
|
||||
"""Derive tags from component counts."""
|
||||
tag_map = {
|
||||
"skills": "skills",
|
||||
"agents": "agents",
|
||||
"commands": "commands",
|
||||
"hooks": "hooks",
|
||||
"mcps": "mcps",
|
||||
"lsps": "lsps",
|
||||
"rules": "rules",
|
||||
}
|
||||
return [tag_map[k] for k in tag_map if components.get(k, 0) > 0]
|
||||
|
||||
|
||||
def get_local_source_path(plugin_entry):
|
||||
"""Extract local source path from a plugin entry, or None if external."""
|
||||
source = plugin_entry.get("source", "")
|
||||
if isinstance(source, str):
|
||||
if source.startswith("./") or source.startswith("../") or source == "./" or source == ".":
|
||||
return source.rstrip("/")
|
||||
# Could be a short-form like "owner/repo" — that's external
|
||||
if source.startswith("http") or source.endswith(".git") or "/" in source:
|
||||
return None
|
||||
return None
|
||||
elif isinstance(source, dict):
|
||||
src_type = source.get("source", "")
|
||||
if src_type in ("url", "git-subdir", "github"):
|
||||
return None
|
||||
# Local path in source dict
|
||||
path = source.get("path", "")
|
||||
if path.startswith("./"):
|
||||
return path.rstrip("/")
|
||||
return None
|
||||
|
||||
|
||||
def _strip_component_ext(name):
|
||||
for ext in (".md", ".json", ".js", ".ts", ".py"):
|
||||
if name.endswith(ext):
|
||||
return name[: -len(ext)]
|
||||
return name
|
||||
|
||||
|
||||
def extract_frontmatter_description(content):
|
||||
"""Pull a `description:` value out of a markdown file's YAML frontmatter."""
|
||||
if not content or not content.startswith("---"):
|
||||
return ""
|
||||
end = content.find("---", 3)
|
||||
if end == -1:
|
||||
return ""
|
||||
frontmatter = content[3:end]
|
||||
for line in frontmatter.split("\n"):
|
||||
line = line.strip()
|
||||
if line.startswith("description:"):
|
||||
desc = line.split("description:", 1)[1].strip()
|
||||
if len(desc) >= 2 and desc[0] == desc[-1] and desc[0] in ("'", '"'):
|
||||
desc = desc[1:-1]
|
||||
return desc
|
||||
return ""
|
||||
|
||||
|
||||
def scan_plugin_dir_components(repo, dir_path, fetch_descriptions=True):
|
||||
"""Scan a plugin directory for skills/, agents/, commands/, hooks/ subdirs
|
||||
and list the files inside each. Returns (counts, items) where `counts` is
|
||||
a dict of type -> count and `items` is a dict of type -> list of
|
||||
{"name", "description"} dicts, for types where names were resolved."""
|
||||
components = {}
|
||||
component_items = {}
|
||||
# List the plugin's root directory
|
||||
listing = gh_dir_listing(repo, dir_path)
|
||||
if not listing:
|
||||
return components, component_items
|
||||
|
||||
dir_names = {item["name"]: item["type"] for item in listing if isinstance(item, dict)}
|
||||
component_dirs = ["skills", "agents", "commands", "hooks"]
|
||||
|
||||
for comp_dir in component_dirs:
|
||||
if comp_dir in dir_names and dir_names[comp_dir] == "dir":
|
||||
# List items inside this component directory
|
||||
items = gh_dir_listing(repo, f"{dir_path}/{comp_dir}")
|
||||
if items:
|
||||
entries = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if item["type"] == "dir":
|
||||
item_name = item["name"]
|
||||
description = ""
|
||||
if fetch_descriptions:
|
||||
md = gh_file_content(repo, f"{dir_path}/{comp_dir}/{item_name}/SKILL.md")
|
||||
if not md:
|
||||
md = gh_file_content(repo, f"{dir_path}/{comp_dir}/{item_name}/{item_name}.md")
|
||||
description = extract_frontmatter_description(md)
|
||||
entries.append({"name": item_name, "description": description})
|
||||
elif item["name"].endswith((".md", ".json", ".js", ".ts", ".py")):
|
||||
item_name = _strip_component_ext(item["name"])
|
||||
description = ""
|
||||
if fetch_descriptions and item["name"].endswith(".md"):
|
||||
md = gh_file_content(repo, f"{dir_path}/{comp_dir}/{item['name']}")
|
||||
description = extract_frontmatter_description(md)
|
||||
entries.append({"name": item_name, "description": description})
|
||||
if entries:
|
||||
components[comp_dir] = len(entries)
|
||||
component_items[comp_dir] = entries
|
||||
|
||||
# Check for mcpServers in plugin.json
|
||||
plugin_json_raw = gh_file_content(repo, f"{dir_path}/.claude-plugin/plugin.json")
|
||||
if plugin_json_raw:
|
||||
pj = parse_json_safe(plugin_json_raw)
|
||||
if pj:
|
||||
mcps = pj.get("mcpServers")
|
||||
if isinstance(mcps, dict) and len(mcps) > 0:
|
||||
components["mcps"] = len(mcps)
|
||||
component_items["mcps"] = [{"name": k, "description": ""} for k in mcps.keys()]
|
||||
lsps = pj.get("lspServers")
|
||||
if isinstance(lsps, dict) and len(lsps) > 0:
|
||||
components["lsps"] = len(lsps)
|
||||
component_items["lsps"] = [{"name": k, "description": ""} for k in lsps.keys()]
|
||||
|
||||
return components, component_items
|
||||
|
||||
|
||||
def extract_marketplace_plugins_detail(marketplace, repo=None, scan_local=True):
|
||||
"""Extract detailed plugin list from marketplace.json.
|
||||
If scan_local=True and repo is provided, scan local plugin directories
|
||||
for actual component counts."""
|
||||
if not marketplace:
|
||||
return []
|
||||
plugins_list = marketplace.get("plugins", [])
|
||||
details = []
|
||||
local_scan_count = 0
|
||||
max_local_scans = 50 # Limit API calls
|
||||
|
||||
for p in plugins_list:
|
||||
entry = {
|
||||
"name": p.get("name", "unknown"),
|
||||
"description": p.get("description", ""),
|
||||
}
|
||||
# Extract source info
|
||||
source = p.get("source", "")
|
||||
local_path = get_local_source_path(p)
|
||||
|
||||
if isinstance(source, str):
|
||||
entry["source"] = source
|
||||
elif isinstance(source, dict):
|
||||
entry["source"] = source.get("url", source.get("repo", ""))
|
||||
if source.get("path"):
|
||||
entry["source_path"] = source["path"]
|
||||
|
||||
# Extract author if present
|
||||
author = p.get("author")
|
||||
if isinstance(author, dict):
|
||||
entry["author"] = author.get("name", "")
|
||||
elif isinstance(author, str):
|
||||
entry["author"] = author
|
||||
|
||||
# Tags/keywords
|
||||
tags = p.get("tags", p.get("keywords", []))
|
||||
if tags:
|
||||
entry["tags"] = tags
|
||||
|
||||
# Inline components from marketplace.json
|
||||
components = {}
|
||||
component_items = {}
|
||||
can_fetch_local_desc = bool(local_path and repo and scan_local and local_scan_count < max_local_scans)
|
||||
used_local_desc_fetch = False
|
||||
for key in ("skills", "agents", "commands"):
|
||||
val = p.get(key)
|
||||
if isinstance(val, list) and len(val) > 0:
|
||||
components[key] = len(val)
|
||||
entries = []
|
||||
for v in val:
|
||||
if isinstance(v, str):
|
||||
base = v.rstrip("/").replace("\\", "/").split("/")[-1]
|
||||
item_name = _strip_component_ext(base)
|
||||
description = ""
|
||||
if can_fetch_local_desc and v.endswith(".md"):
|
||||
rel = v.lstrip("./")
|
||||
md = gh_file_content(repo, f"{local_path.lstrip('./')}/{rel}")
|
||||
description = extract_frontmatter_description(md)
|
||||
used_local_desc_fetch = True
|
||||
entries.append({"name": item_name, "description": description})
|
||||
elif isinstance(v, dict):
|
||||
n = v.get("name") or v.get("path", "")
|
||||
if n:
|
||||
item_name = _strip_component_ext(str(n).rstrip("/").split("/")[-1])
|
||||
entries.append({"name": item_name, "description": v.get("description", "")})
|
||||
if entries:
|
||||
component_items[key] = entries
|
||||
mcps = p.get("mcpServers")
|
||||
if isinstance(mcps, dict) and len(mcps) > 0:
|
||||
components["mcps"] = len(mcps)
|
||||
component_items["mcps"] = [{"name": k, "description": ""} for k in mcps.keys()]
|
||||
lsps = p.get("lspServers")
|
||||
if isinstance(lsps, dict) and len(lsps) > 0:
|
||||
components["lsps"] = len(lsps)
|
||||
component_items["lsps"] = [{"name": k, "description": ""} for k in lsps.keys()]
|
||||
|
||||
if used_local_desc_fetch:
|
||||
local_scan_count += 1
|
||||
|
||||
# If no inline components and this is a local plugin, scan its directory
|
||||
if not components and local_path and repo and scan_local and local_scan_count < max_local_scans:
|
||||
scanned_counts, scanned_items = scan_plugin_dir_components(repo, local_path.lstrip("./"))
|
||||
if scanned_counts:
|
||||
components = scanned_counts
|
||||
component_items = scanned_items
|
||||
local_scan_count += 1
|
||||
|
||||
if components:
|
||||
entry["components"] = components
|
||||
if component_items:
|
||||
entry["components_items"] = component_items
|
||||
|
||||
details.append(entry)
|
||||
return details
|
||||
|
||||
|
||||
def aggregate_marketplace_components(plugins_detail):
|
||||
"""Sum component counts across all plugins in a marketplace."""
|
||||
totals = {}
|
||||
for p in plugins_detail:
|
||||
for key, count in p.get("components", {}).items():
|
||||
totals[key] = totals.get(key, 0) + count
|
||||
return totals
|
||||
|
||||
|
||||
def build_highlights(marketplace, plugin_json, repo_type, repo_info):
|
||||
"""Auto-generate highlights from real data."""
|
||||
highlights = []
|
||||
owner = repo_info.get("owner", {}).get("login", "")
|
||||
|
||||
if owner == "anthropics":
|
||||
highlights.append("Official Anthropic repository")
|
||||
elif owner == "hashicorp":
|
||||
highlights.append("Official HashiCorp release")
|
||||
elif owner == "timescale":
|
||||
highlights.append("Official Timescale plugin")
|
||||
|
||||
if marketplace:
|
||||
plugins_list = marketplace.get("plugins", [])
|
||||
if len(plugins_list) > 1:
|
||||
highlights.append(f"{len(plugins_list)} plugins in marketplace")
|
||||
|
||||
if plugin_json:
|
||||
parts = []
|
||||
for key, label in [("skills", "skills"), ("agents", "agents"), ("commands", "commands"),
|
||||
("hooks", "hooks"), ("mcpServers", "MCPs"), ("lspServers", "LSPs")]:
|
||||
val = plugin_json.get(key)
|
||||
if isinstance(val, list) and len(val) > 0:
|
||||
parts.append(f"{len(val)} {label}")
|
||||
elif isinstance(val, dict) and len(val) > 0:
|
||||
parts.append(f"{len(val)} {label}")
|
||||
if parts:
|
||||
highlights.append("Plugin declares: " + ", ".join(parts))
|
||||
|
||||
# Website
|
||||
website = repo_info.get("homepage")
|
||||
if website and website.startswith("http"):
|
||||
highlights.append(f"Web UI: {website}")
|
||||
|
||||
return highlights[:3] # Max 3
|
||||
|
||||
|
||||
def process_repo(repo_full, website_override=None):
|
||||
"""Process a single repo and return its plugin data."""
|
||||
owner, name = repo_full.split("/")
|
||||
print(f"\n📦 Processing {repo_full}...")
|
||||
|
||||
# 1. Get repo info
|
||||
repo_info = gh_api(f"repos/{repo_full}")
|
||||
if not repo_info:
|
||||
print(f" ❌ Repo not found")
|
||||
return None
|
||||
|
||||
stars = repo_info.get("stargazers_count", 0)
|
||||
description = repo_info.get("description", "")
|
||||
repo_owner = repo_info.get("owner", {}).get("login", owner)
|
||||
|
||||
# 2. Read marketplace.json
|
||||
marketplace_raw = gh_file_content(repo_full, ".claude-plugin/marketplace.json")
|
||||
marketplace = parse_json_safe(marketplace_raw)
|
||||
if not marketplace:
|
||||
print(f" ⚠️ No valid marketplace.json")
|
||||
return None
|
||||
|
||||
# 3. Read plugin.json (optional)
|
||||
plugin_raw = gh_file_content(repo_full, ".claude-plugin/plugin.json")
|
||||
plugin_json = parse_json_safe(plugin_raw)
|
||||
|
||||
# 4. Classify type
|
||||
repo_type = classify_repo_type(marketplace, plugin_json)
|
||||
print(f" Type: {repo_type} | Stars: {stars:,}")
|
||||
|
||||
# 5. Extract marketplace plugin details (scan local dirs for component counts)
|
||||
plugins_detail = extract_marketplace_plugins_detail(marketplace, repo=repo_full, scan_local=True)
|
||||
marketplace_component_totals = aggregate_marketplace_components(plugins_detail)
|
||||
|
||||
# 6. Extract single plugin components
|
||||
single_components = extract_plugin_components(plugin_json) if plugin_json else {}
|
||||
|
||||
# 7. Build contains
|
||||
if repo_type == "marketplace":
|
||||
contains = {"plugins": len(marketplace.get("plugins", []))}
|
||||
if marketplace_component_totals:
|
||||
contains["components"] = marketplace_component_totals
|
||||
else:
|
||||
contains = single_components if single_components else {}
|
||||
|
||||
# 8. Build tags
|
||||
all_components = {**marketplace_component_totals, **single_components}
|
||||
tags = extract_tags_from_components(all_components)
|
||||
# If no tags from components, infer from plugin descriptions and marketplace data
|
||||
if not tags:
|
||||
tag_set = set()
|
||||
# Check plugin entry tags/keywords
|
||||
for p in plugins_detail:
|
||||
for t in p.get("tags", []):
|
||||
t_lower = t.lower()
|
||||
if t_lower in ("skills", "agents", "commands", "hooks", "mcps", "lsps"):
|
||||
tag_set.add(t_lower)
|
||||
# Scan descriptions for clues
|
||||
all_text = " ".join(p.get("description", "") for p in plugins_detail).lower()
|
||||
all_text += " " + description.lower()
|
||||
keyword_map = {
|
||||
"skill": "skills", "agent": "agents", "subagent": "agents",
|
||||
"command": "commands", "slash command": "commands",
|
||||
"hook": "hooks", "mcp": "mcps", "lsp": "lsps",
|
||||
}
|
||||
for keyword, tag in keyword_map.items():
|
||||
if keyword in all_text:
|
||||
tag_set.add(tag)
|
||||
tags = sorted(tag_set)
|
||||
|
||||
# 9. Build highlights
|
||||
highlights = build_highlights(marketplace, plugin_json, repo_type, repo_info)
|
||||
|
||||
# 10. Determine name — prefer repo name humanized over marketplace slug
|
||||
repo_display = repo_info.get("name", name)
|
||||
mp_name = marketplace.get("name", "")
|
||||
plugin_name = plugin_json.get("name", "") if plugin_json else ""
|
||||
# Use marketplace/plugin name only if it looks human-readable (has spaces or capitals)
|
||||
if mp_name and (mp_name != mp_name.lower().replace(" ", "-") or " " in mp_name):
|
||||
display_name = mp_name
|
||||
elif plugin_name and (plugin_name != plugin_name.lower().replace(" ", "-") or " " in plugin_name):
|
||||
display_name = plugin_name
|
||||
else:
|
||||
display_name = repo_display.replace("-", " ").title()
|
||||
|
||||
# 11. Slug
|
||||
slug = name.lower()
|
||||
# Avoid collisions by prepending owner for common names
|
||||
if slug in ("claude-skills", "awesome-claude-skills", "awesome-claude-code-plugins"):
|
||||
slug = f"{owner.lower()}-{slug}"
|
||||
|
||||
# 12. Website
|
||||
website = website_override or repo_info.get("homepage") or None
|
||||
if website and not website.startswith("http"):
|
||||
website = None
|
||||
|
||||
# 13. Build result
|
||||
result = {
|
||||
"slug": slug,
|
||||
"name": display_name,
|
||||
"author": repo_owner,
|
||||
"description": description or marketplace.get("description", ""),
|
||||
"github": f"https://github.com/{repo_full}",
|
||||
"stars": stars,
|
||||
"type": repo_type,
|
||||
"tags": tags,
|
||||
"contains": contains,
|
||||
"highlights": highlights,
|
||||
}
|
||||
|
||||
if website:
|
||||
result["website"] = website
|
||||
|
||||
# 14. Add marketplace plugins list (for individual pages)
|
||||
if repo_type == "marketplace" and plugins_detail:
|
||||
result["plugins_list"] = plugins_detail
|
||||
|
||||
# 15. Add plugin.json details for single plugins
|
||||
if plugin_json and repo_type == "plugin":
|
||||
plugin_detail = {}
|
||||
for key in ("skills", "agents", "commands"):
|
||||
val = plugin_json.get(key)
|
||||
if isinstance(val, list) and val:
|
||||
plugin_detail[key] = val
|
||||
mcps = plugin_json.get("mcpServers")
|
||||
if isinstance(mcps, dict) and mcps:
|
||||
plugin_detail["mcpServers"] = list(mcps.keys())
|
||||
lsps = plugin_json.get("lspServers")
|
||||
if isinstance(lsps, dict) and lsps:
|
||||
plugin_detail["lspServers"] = list(lsps.keys())
|
||||
if plugin_detail:
|
||||
result["plugin_manifest"] = plugin_detail
|
||||
|
||||
print(f" ✅ {display_name} — {repo_type} — {stars:,} stars — tags: {tags}")
|
||||
return result
|
||||
|
||||
|
||||
# --- Main ---
|
||||
|
||||
def main():
|
||||
print("🔌 Generating plugins.json from GitHub .claude-plugin/ data...\n")
|
||||
|
||||
# Check gh CLI is available
|
||||
try:
|
||||
test = subprocess.run(
|
||||
["/usr/local/bin/gh", "api", "rate_limit", "--jq", ".rate.remaining"],
|
||||
capture_output=True, text=True, timeout=10, env=_GH_ENV
|
||||
)
|
||||
if test.returncode == 0:
|
||||
print(f"✅ gh CLI authenticated. API requests remaining: {test.stdout.strip()}\n")
|
||||
else:
|
||||
print("⚠️ gh CLI may not be authenticated. Run `gh auth login` first.\n")
|
||||
except FileNotFoundError:
|
||||
print("❌ gh CLI not found. Install it: https://cli.github.com\n")
|
||||
sys.exit(1)
|
||||
|
||||
results = []
|
||||
errors = []
|
||||
|
||||
for repo_full, website in REPOS:
|
||||
try:
|
||||
data = process_repo(repo_full, website)
|
||||
if data:
|
||||
results.append(data)
|
||||
else:
|
||||
errors.append(repo_full)
|
||||
except Exception as e:
|
||||
print(f" ❌ Error processing {repo_full}: {e}")
|
||||
errors.append(repo_full)
|
||||
|
||||
# Sort by stars descending
|
||||
results.sort(key=lambda x: x["stars"], reverse=True)
|
||||
|
||||
# Write output
|
||||
output_path = os.path.abspath(OUTPUT_PATH)
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(results, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"✅ Generated {output_path}")
|
||||
print(f" {len(results)} plugins/marketplaces written")
|
||||
print(f" {len(errors)} errors: {errors}" if errors else " 0 errors")
|
||||
print(f" Total API requests: {_request_count}")
|
||||
|
||||
# Summary
|
||||
marketplaces = [r for r in results if r["type"] == "marketplace"]
|
||||
plugins = [r for r in results if r["type"] == "plugin"]
|
||||
print(f"\n 📊 {len(marketplaces)} marketplaces | {len(plugins)} individual plugins")
|
||||
total_stars = sum(r["stars"] for r in results)
|
||||
print(f" ⭐ {total_stars:,} total stars across all repos")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a self-hosted "Stargazers over time" SVG chart.
|
||||
|
||||
GitHub now restricts the stargazers endpoint to a repository's own admins and
|
||||
collaborators, so third-party live-chart services (star-history free tier,
|
||||
starchart.cc, ...) return "Requires authentication" for everyone. This script
|
||||
fetches the star data with an authenticated token (in CI, the repo's own
|
||||
GITHUB_TOKEN works because it can read the repo's own stargazers) and renders a
|
||||
static, theme-aware SVG committed to docs/star-history.svg.
|
||||
|
||||
Usage:
|
||||
GITHUB_TOKEN=xxx python scripts/generate_star_history.py
|
||||
# local test:
|
||||
GITHUB_TOKEN=$(gh auth token) python scripts/generate_star_history.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import requests
|
||||
|
||||
REPO = os.environ.get("STAR_HISTORY_REPO", "davila7/claude-code-templates")
|
||||
OUTPUT = os.environ.get("STAR_HISTORY_OUTPUT", "docs/star-history.svg")
|
||||
PER_PAGE = 100
|
||||
MAX_PAGES = 400 # GitHub caps stargazers pagination at 400 pages (40k stars)
|
||||
|
||||
WIDTH, HEIGHT = 800, 400
|
||||
PAD_L, PAD_R, PAD_T, PAD_B = 70, 55, 50, 55
|
||||
|
||||
|
||||
def fetch_stargazers(token):
|
||||
"""Return a sorted list of datetime objects, one per star."""
|
||||
session = requests.Session()
|
||||
session.headers.update({
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/vnd.github.star+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
"User-Agent": "claude-code-templates-star-history",
|
||||
})
|
||||
|
||||
dates = []
|
||||
for page in range(1, MAX_PAGES + 1):
|
||||
resp = session.get(
|
||||
f"https://api.github.com/repos/{REPO}/stargazers",
|
||||
params={"per_page": PER_PAGE, "page": page},
|
||||
timeout=30,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise SystemExit(
|
||||
f"GitHub API error {resp.status_code} on page {page}: {resp.text[:200]}"
|
||||
)
|
||||
batch = resp.json()
|
||||
if not batch:
|
||||
break
|
||||
for entry in batch:
|
||||
starred_at = entry.get("starred_at")
|
||||
if starred_at:
|
||||
dates.append(
|
||||
datetime.strptime(starred_at, "%Y-%m-%dT%H:%M:%SZ").replace(
|
||||
tzinfo=timezone.utc
|
||||
)
|
||||
)
|
||||
if len(batch) < PER_PAGE:
|
||||
break
|
||||
|
||||
dates.sort()
|
||||
return dates
|
||||
|
||||
|
||||
def build_series(dates, max_points=100):
|
||||
"""Cumulative (timestamp, count) points, downsampled to max_points."""
|
||||
total = len(dates)
|
||||
if total == 0:
|
||||
return []
|
||||
points = [(d, i + 1) for i, d in enumerate(dates)]
|
||||
if total <= max_points:
|
||||
return points
|
||||
step = total / max_points
|
||||
sampled = [points[int(i * step)] for i in range(max_points)]
|
||||
sampled.append(points[-1]) # always include the latest point
|
||||
return sampled
|
||||
|
||||
|
||||
def esc(text):
|
||||
return (
|
||||
str(text)
|
||||
.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
)
|
||||
|
||||
|
||||
def render_svg(series, repo):
|
||||
if not series:
|
||||
raise SystemExit("No stargazer data to render.")
|
||||
|
||||
t_min = series[0][0].timestamp()
|
||||
t_max = series[-1][0].timestamp()
|
||||
c_max = series[-1][1]
|
||||
t_span = max(t_max - t_min, 1)
|
||||
|
||||
plot_w = WIDTH - PAD_L - PAD_R
|
||||
plot_h = HEIGHT - PAD_T - PAD_B
|
||||
|
||||
def x(ts):
|
||||
return PAD_L + (ts - t_min) / t_span * plot_w
|
||||
|
||||
def y(count):
|
||||
return PAD_T + plot_h - (count / c_max) * plot_h
|
||||
|
||||
pts = [(x(ts.timestamp()), y(c)) for ts, c in series]
|
||||
line = " ".join(f"{px:.1f},{py:.1f}" for px, py in pts)
|
||||
area = (
|
||||
f"{PAD_L:.1f},{PAD_T + plot_h:.1f} "
|
||||
+ line
|
||||
+ f" {pts[-1][0]:.1f},{PAD_T + plot_h:.1f}"
|
||||
)
|
||||
|
||||
# Y axis ticks (5 gridlines)
|
||||
y_ticks = []
|
||||
for i in range(5):
|
||||
val = round(c_max * i / 4)
|
||||
y_ticks.append((y(val), val))
|
||||
|
||||
# X axis ticks (dates)
|
||||
x_ticks = []
|
||||
for i in range(5):
|
||||
ts = t_min + t_span * i / 4
|
||||
label = datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%b %Y")
|
||||
x_ticks.append((x(ts), label))
|
||||
|
||||
grid_lines = ""
|
||||
for gy, val in y_ticks:
|
||||
grid_lines += (
|
||||
f'<line x1="{PAD_L}" y1="{gy:.1f}" x2="{WIDTH - PAD_R}" y2="{gy:.1f}" '
|
||||
f'class="grid"/>\n'
|
||||
f'<text x="{PAD_L - 10}" y="{gy + 4:.1f}" text-anchor="end" '
|
||||
f'class="tick">{val:,}</text>\n'
|
||||
)
|
||||
x_labels = ""
|
||||
for gx, label in x_ticks:
|
||||
x_labels += (
|
||||
f'<text x="{gx:.1f}" y="{HEIGHT - PAD_B + 22}" text-anchor="middle" '
|
||||
f'class="tick">{esc(label)}</text>\n'
|
||||
)
|
||||
|
||||
updated = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
return f'''<svg xmlns="http://www.w3.org/2000/svg" width="{WIDTH}" height="{HEIGHT}" viewBox="0 0 {WIDTH} {HEIGHT}" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif">
|
||||
<style>
|
||||
.bg {{ fill: #ffffff; }}
|
||||
.title {{ fill: #1f2328; font-size: 17px; font-weight: 600; }}
|
||||
.subtitle {{ fill: #656d76; font-size: 11px; }}
|
||||
.grid {{ stroke: #d0d7de; stroke-width: 1; stroke-dasharray: 3 3; }}
|
||||
.axis {{ stroke: #656d76; stroke-width: 1.5; }}
|
||||
.tick {{ fill: #656d76; font-size: 11px; }}
|
||||
.area {{ fill: #ffd33d; opacity: 0.18; }}
|
||||
.line {{ stroke: #f0b400; stroke-width: 2.5; fill: none; stroke-linejoin: round; stroke-linecap: round; }}
|
||||
.dot {{ fill: #f0b400; }}
|
||||
@media (prefers-color-scheme: dark) {{
|
||||
.bg {{ fill: #0d1117; }}
|
||||
.title {{ fill: #e6edf3; }}
|
||||
.subtitle {{ fill: #8b949e; }}
|
||||
.grid {{ stroke: #30363d; }}
|
||||
.axis {{ stroke: #8b949e; }}
|
||||
.tick {{ fill: #8b949e; }}
|
||||
}}
|
||||
</style>
|
||||
<rect class="bg" width="{WIDTH}" height="{HEIGHT}" rx="6"/>
|
||||
<text x="{PAD_L}" y="26" class="title">Stargazers over time</text>
|
||||
<text x="{PAD_L}" y="42" class="subtitle">{esc(repo)} · {c_max:,} stars · updated {updated}</text>
|
||||
{grid_lines}
|
||||
<line x1="{PAD_L}" y1="{PAD_T}" x2="{PAD_L}" y2="{PAD_T + plot_h}" class="axis"/>
|
||||
<line x1="{PAD_L}" y1="{PAD_T + plot_h}" x2="{WIDTH - PAD_R}" y2="{PAD_T + plot_h}" class="axis"/>
|
||||
<polygon class="area" points="{area}"/>
|
||||
<polyline class="line" points="{line}"/>
|
||||
<circle class="dot" cx="{pts[-1][0]:.1f}" cy="{pts[-1][1]:.1f}" r="4"/>
|
||||
{x_labels}
|
||||
</svg>
|
||||
'''
|
||||
|
||||
|
||||
def main():
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if not token:
|
||||
raise SystemExit("GITHUB_TOKEN is required (repo read access to stargazers).")
|
||||
|
||||
print(f"Fetching stargazers for {REPO} ...", file=sys.stderr)
|
||||
dates = fetch_stargazers(token)
|
||||
print(f"Got {len(dates)} stars.", file=sys.stderr)
|
||||
|
||||
series = build_series(dates)
|
||||
svg = render_svg(series, REPO)
|
||||
|
||||
os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
|
||||
with open(OUTPUT, "w", encoding="utf-8") as fh:
|
||||
fh.write(svg)
|
||||
print(f"Wrote {OUTPUT} ({len(svg)} bytes).", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+866
@@ -0,0 +1,866 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Trending Data Generator Script
|
||||
Fetches download data from Supabase and generates trending-data.json for the Claude Code Templates project.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import requests
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from collections import defaultdict, Counter
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables
|
||||
load_dotenv()
|
||||
|
||||
def fetch_with_retry(url, headers, max_retries=5, timeout=60):
|
||||
"""
|
||||
Fetch data from API with retry logic and exponential backoff.
|
||||
Handles 500, 503, timeouts, and connection errors with aggressive retries.
|
||||
|
||||
Args:
|
||||
url: The URL to fetch
|
||||
headers: Request headers
|
||||
max_retries: Maximum number of retry attempts
|
||||
timeout: Request timeout in seconds
|
||||
|
||||
Returns:
|
||||
Response object or None if all retries failed
|
||||
"""
|
||||
retryable_statuses = {500, 502, 503, 504}
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = requests.get(url, headers=headers, timeout=timeout)
|
||||
|
||||
# Return successful responses
|
||||
if response.status_code in [200, 206]:
|
||||
return response
|
||||
|
||||
# Retry on server errors with exponential backoff
|
||||
if response.status_code in retryable_statuses:
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = (2 ** attempt) * 3 # 3s, 6s, 12s, 24s
|
||||
print(f"⏳ Server error {response.status_code} on attempt {attempt + 1}/{max_retries}. Retrying in {wait_time}s...")
|
||||
time.sleep(wait_time)
|
||||
continue
|
||||
else:
|
||||
print(f"⚠️ Server error {response.status_code} after {max_retries} attempts")
|
||||
return None
|
||||
|
||||
# For non-retryable errors, return immediately
|
||||
print(f"⚠️ API returned status {response.status_code}: {response.text[:200]}")
|
||||
return None
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = (2 ** attempt) * 3
|
||||
print(f"⏳ Request timeout on attempt {attempt + 1}/{max_retries}. Retrying in {wait_time}s...")
|
||||
time.sleep(wait_time)
|
||||
continue
|
||||
else:
|
||||
print(f"❌ Request timed out after {max_retries} attempts")
|
||||
return None
|
||||
|
||||
except requests.exceptions.ConnectionError:
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = (2 ** attempt) * 3
|
||||
print(f"⏳ Connection error on attempt {attempt + 1}/{max_retries}. Retrying in {wait_time}s...")
|
||||
time.sleep(wait_time)
|
||||
continue
|
||||
else:
|
||||
print(f"❌ Connection failed after {max_retries} attempts")
|
||||
return None
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"❌ Request error: {str(e)}")
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def main():
|
||||
"""Main function to generate trending data"""
|
||||
print("🚀 Generating trending data from Supabase...")
|
||||
|
||||
# Get Supabase credentials
|
||||
supabase_url = os.getenv("SUPABASE_URL")
|
||||
supabase_api_key = os.getenv("SUPABASE_API_KEY")
|
||||
|
||||
if not supabase_url or not supabase_api_key:
|
||||
print("❌ Error: Missing Supabase credentials in .env file")
|
||||
return
|
||||
|
||||
try:
|
||||
# Fetch all component downloads using REST API
|
||||
print("📊 Fetching download data from Supabase...")
|
||||
|
||||
headers = {
|
||||
'apikey': supabase_api_key,
|
||||
'Authorization': f'Bearer {supabase_api_key}',
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
# Get total count first
|
||||
count_url = f"{supabase_url}/rest/v1/component_downloads"
|
||||
count_headers = {**headers, 'Prefer': 'count=exact'}
|
||||
count_response = requests.head(count_url, headers=count_headers)
|
||||
|
||||
total_count = 0
|
||||
if 'content-range' in count_response.headers:
|
||||
total_count = int(count_response.headers['content-range'].split('/')[-1])
|
||||
|
||||
print(f"📊 Total records in database: {total_count}")
|
||||
|
||||
# Fetch ALL data using cursor-based pagination
|
||||
all_downloads = []
|
||||
page_size = 1000
|
||||
last_id = 0
|
||||
page_num = 0
|
||||
consecutive_errors = 0
|
||||
max_consecutive_errors = 3
|
||||
|
||||
print("📊 Using cursor-based pagination to fetch all records...")
|
||||
|
||||
while True:
|
||||
page_num += 1
|
||||
|
||||
api_url = f"{supabase_url}/rest/v1/component_downloads?id=gt.{last_id}&order=id.asc&limit={page_size}"
|
||||
|
||||
response = fetch_with_retry(api_url, headers, max_retries=5, timeout=60)
|
||||
|
||||
if response is None:
|
||||
consecutive_errors += 1
|
||||
if consecutive_errors >= max_consecutive_errors:
|
||||
print(f"⚠️ {max_consecutive_errors} consecutive failures. Stopping at {len(all_downloads):,} records.")
|
||||
break
|
||||
# Skip ahead by estimating next ID range to recover from persistent errors
|
||||
last_id += page_size
|
||||
print(f"⚠️ Skipping ahead to id > {last_id} (attempt {consecutive_errors}/{max_consecutive_errors})")
|
||||
time.sleep(5)
|
||||
continue
|
||||
|
||||
page_data = response.json()
|
||||
if not page_data:
|
||||
print(f"✅ Reached end of data at page {page_num}")
|
||||
break
|
||||
|
||||
consecutive_errors = 0 # Reset on success
|
||||
all_downloads.extend(page_data)
|
||||
last_id = page_data[-1]['id']
|
||||
|
||||
# Progress indicator every 50 pages
|
||||
if page_num % 50 == 0:
|
||||
pct = (len(all_downloads) / total_count * 100) if total_count > 0 else 0
|
||||
print(f"📄 Page {page_num}: {len(all_downloads):,}/{total_count:,} records ({pct:.1f}%)")
|
||||
|
||||
if len(page_data) < page_size:
|
||||
print(f"✅ Fetched final page {page_num} with {len(page_data)} records")
|
||||
break
|
||||
|
||||
if len(all_downloads) >= 10000000:
|
||||
print(f"⚠️ Reached safety limit of 10,000,000 records")
|
||||
break
|
||||
|
||||
if not all_downloads:
|
||||
print("❌ No data fetched from Supabase")
|
||||
print("📝 Generating fallback trending data...")
|
||||
trending_data = generate_fallback_trending_data()
|
||||
else:
|
||||
print(f"\n✅ Successfully fetched {len(all_downloads):,} total records from Supabase")
|
||||
print(f"📊 Processing download data to generate trending statistics...")
|
||||
# Process the real data
|
||||
trending_data = process_downloads_data(all_downloads)
|
||||
|
||||
# Write to JSON file
|
||||
output_file = "docs/trending-data.json"
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(trending_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"✅ Successfully generated {output_file}")
|
||||
print(f"📊 Statistics:")
|
||||
for component_type, items in trending_data['trending'].items():
|
||||
print(f" • {component_type}: {len(items)} items")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {str(e)}")
|
||||
print("📝 Generating fallback trending data...")
|
||||
trending_data = generate_fallback_trending_data()
|
||||
|
||||
# Write fallback data
|
||||
output_file = "docs/trending-data.json"
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(trending_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"✅ Generated fallback {output_file}")
|
||||
|
||||
def process_downloads_data(downloads):
|
||||
"""Process raw download data and generate trending structure"""
|
||||
|
||||
# Components to exclude from trending (test/internal components)
|
||||
EXCLUDED_COMPONENTS = {
|
||||
'test-command', 'test-agent', 'test-setting', 'test-hook',
|
||||
'test-mcp', 'test-skill', 'test-template', 'test-from-production',
|
||||
'test-component', 'test', 'demo-component', 'example-component'
|
||||
}
|
||||
|
||||
# Calculate date ranges with timezone awareness
|
||||
now = datetime.now(timezone.utc)
|
||||
today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
week_start = today_start - timedelta(days=7)
|
||||
month_start = today_start - timedelta(days=30)
|
||||
|
||||
# For charting - collect daily data for the last 30 days
|
||||
chart_data = defaultdict(lambda: defaultdict(int)) # {date: {category: count}}
|
||||
|
||||
# Group downloads by component
|
||||
component_stats = defaultdict(lambda: {
|
||||
'total': 0,
|
||||
'today': 0,
|
||||
'week': 0,
|
||||
'month': 0,
|
||||
'component_type': '',
|
||||
'category': '',
|
||||
'name': ''
|
||||
})
|
||||
|
||||
# Track unique countries
|
||||
unique_countries = set()
|
||||
# Track downloads by country
|
||||
country_downloads = Counter()
|
||||
|
||||
# Debug: Show sample of downloads and date ranges
|
||||
print(f"🔍 Processing {len(downloads)} downloads...")
|
||||
print(f"📄 Sample download record: {downloads[0] if downloads else 'None'}")
|
||||
print(f"📅 Date ranges being used:")
|
||||
print(f" • now: {now}")
|
||||
print(f" • today_start: {today_start}")
|
||||
print(f" • week_start: {week_start}")
|
||||
print(f" • month_start: {month_start}")
|
||||
|
||||
# Debug: Track downloads by period
|
||||
today_count = 0
|
||||
week_count = 0
|
||||
month_count = 0
|
||||
|
||||
for download in downloads:
|
||||
# Parse download timestamp with proper timezone handling
|
||||
timestamp_str = download['download_timestamp']
|
||||
if timestamp_str.endswith('Z'):
|
||||
timestamp_str = timestamp_str.replace('Z', '+00:00')
|
||||
elif '+' not in timestamp_str and '-' not in timestamp_str[-6:]:
|
||||
# No timezone info, assume UTC
|
||||
timestamp_str = timestamp_str + '+00:00'
|
||||
|
||||
try:
|
||||
download_time = datetime.fromisoformat(timestamp_str)
|
||||
# Convert to UTC if not already
|
||||
if download_time.tzinfo is None:
|
||||
download_time = download_time.replace(tzinfo=timezone.utc)
|
||||
except:
|
||||
# Fallback to current time if parsing fails
|
||||
download_time = datetime.now(timezone.utc)
|
||||
|
||||
# Create key that matches generate_components_json.py structure
|
||||
# The key should match format: component_type/category/name
|
||||
category = download.get('category', 'general')
|
||||
component_name = download['component_name']
|
||||
component_type = download['component_type']
|
||||
|
||||
# Handle case where component_name already includes category (like "frontend/react-expert")
|
||||
if '/' in component_name:
|
||||
category = component_name.split('/')[0]
|
||||
actual_name = component_name.split('/')[-1]
|
||||
else:
|
||||
actual_name = component_name
|
||||
|
||||
# Skip test/internal components
|
||||
if actual_name.lower() in EXCLUDED_COMPONENTS:
|
||||
continue
|
||||
|
||||
component_key = f"{component_type}-{actual_name}"
|
||||
stats = component_stats[component_key]
|
||||
|
||||
# Set component info
|
||||
stats['name'] = actual_name
|
||||
stats['component_type'] = component_type
|
||||
stats['category'] = category
|
||||
|
||||
# Count downloads by time period
|
||||
stats['total'] += 1
|
||||
|
||||
# Track unique countries
|
||||
country = download.get('country', 'Unknown')
|
||||
if country and country != 'Unknown':
|
||||
unique_countries.add(country)
|
||||
country_downloads[country] += 1
|
||||
|
||||
if download_time >= today_start:
|
||||
stats['today'] += 1
|
||||
today_count += 1
|
||||
|
||||
if download_time >= week_start:
|
||||
stats['week'] += 1
|
||||
week_count += 1
|
||||
|
||||
if download_time >= month_start:
|
||||
stats['month'] += 1
|
||||
month_count += 1
|
||||
|
||||
# Collect daily data for chart (last 30 days only)
|
||||
if download_time >= month_start:
|
||||
download_date = download_time.strftime('%Y-%m-%d')
|
||||
|
||||
# Map component types to plural for consistency
|
||||
type_mapping = {
|
||||
'command': 'commands',
|
||||
'agent': 'agents',
|
||||
'setting': 'settings',
|
||||
'hook': 'hooks',
|
||||
'mcp': 'mcps',
|
||||
'skill': 'skills',
|
||||
'template': 'templates',
|
||||
'plugin': 'plugins',
|
||||
'sandbox': 'sandbox'
|
||||
}
|
||||
mapped_type = type_mapping.get(component_type, component_type + 's')
|
||||
chart_data[download_date][mapped_type] += 1
|
||||
|
||||
# Debug: Print total counts by period
|
||||
print(f"📊 Total downloads by period:")
|
||||
print(f" • Today: {today_count}")
|
||||
print(f" • Week: {week_count}")
|
||||
print(f" • Month: {month_count}")
|
||||
print(f" • Total processed: {len(downloads)}")
|
||||
|
||||
# Debug: Show oldest and newest records
|
||||
if downloads:
|
||||
print(f"📅 Date range in data:")
|
||||
print(f" • Newest: {downloads[0]['download_timestamp']}")
|
||||
print(f" • Oldest: {downloads[-1]['download_timestamp']}")
|
||||
|
||||
# Group by component type and create trending structure
|
||||
trending_by_type = defaultdict(list)
|
||||
|
||||
for component_key, stats in component_stats.items():
|
||||
component_type = stats['component_type']
|
||||
|
||||
trending_item = {
|
||||
'id': component_key.lower().replace(' ', '-'),
|
||||
'name': stats['name'],
|
||||
'category': stats['category'],
|
||||
'downloadsToday': stats['today'],
|
||||
'downloadsWeek': stats['week'],
|
||||
'downloadsMonth': stats['month'],
|
||||
'downloadsTotal': stats['total']
|
||||
}
|
||||
|
||||
trending_by_type[component_type].append(trending_item)
|
||||
|
||||
# Sort each type by weekly downloads (most trending)
|
||||
for component_type in trending_by_type:
|
||||
trending_by_type[component_type].sort(
|
||||
key=lambda x: x['downloadsWeek'],
|
||||
reverse=True
|
||||
)
|
||||
# Keep top 10 for each type
|
||||
trending_by_type[component_type] = trending_by_type[component_type][:10]
|
||||
|
||||
# Process chart data for cumulative growth
|
||||
chart_dates = []
|
||||
chart_categories = ['commands', 'agents', 'settings', 'hooks', 'mcps', 'skills', 'templates']
|
||||
chart_series = {category: [] for category in chart_categories}
|
||||
|
||||
# Generate the last 30 days
|
||||
for i in range(29, -1, -1): # 29 days ago to today
|
||||
date = (today_start - timedelta(days=i)).strftime('%Y-%m-%d')
|
||||
chart_dates.append(date)
|
||||
|
||||
# Calculate cumulative data for each category
|
||||
for category in chart_categories:
|
||||
cumulative = 0
|
||||
for date in chart_dates:
|
||||
daily_count = chart_data.get(date, {}).get(category, 0)
|
||||
cumulative += daily_count
|
||||
chart_series[category].append(cumulative)
|
||||
|
||||
# Calculate global statistics from ALL components (before limiting to top 10)
|
||||
total_components = len(component_stats)
|
||||
total_all_downloads = sum(stats['total'] for stats in component_stats.values())
|
||||
total_month_downloads = sum(stats['month'] for stats in component_stats.values())
|
||||
total_week_downloads = sum(stats['week'] for stats in component_stats.values())
|
||||
total_today_downloads = sum(stats['today'] for stats in component_stats.values())
|
||||
|
||||
print(f"📊 Global Statistics:")
|
||||
print(f" • Total Components: {total_components}")
|
||||
print(f" • Total Downloads: {total_all_downloads:,}")
|
||||
print(f" • Monthly Downloads: {total_month_downloads:,}")
|
||||
print(f" • Weekly Downloads: {total_week_downloads:,}")
|
||||
print(f" • Today Downloads: {total_today_downloads:,}")
|
||||
print(f" • Unique Countries: {len(unique_countries)}")
|
||||
|
||||
# Get top 5 countries by downloads
|
||||
top_countries = country_downloads.most_common(5)
|
||||
|
||||
# Country code to name and flag mapping
|
||||
country_info = {
|
||||
'US': {'name': 'United States', 'flag': '🇺🇸'},
|
||||
'GB': {'name': 'United Kingdom', 'flag': '🇬🇧'},
|
||||
'IN': {'name': 'India', 'flag': '🇮🇳'},
|
||||
'DE': {'name': 'Germany', 'flag': '🇩🇪'},
|
||||
'CA': {'name': 'Canada', 'flag': '🇨🇦'},
|
||||
'FR': {'name': 'France', 'flag': '🇫🇷'},
|
||||
'AU': {'name': 'Australia', 'flag': '🇦🇺'},
|
||||
'JP': {'name': 'Japan', 'flag': '🇯🇵'},
|
||||
'BR': {'name': 'Brazil', 'flag': '🇧🇷'},
|
||||
'ES': {'name': 'Spain', 'flag': '🇪🇸'},
|
||||
'IT': {'name': 'Italy', 'flag': '🇮🇹'},
|
||||
'NL': {'name': 'Netherlands', 'flag': '🇳🇱'},
|
||||
'SE': {'name': 'Sweden', 'flag': '🇸🇪'},
|
||||
'CH': {'name': 'Switzerland', 'flag': '🇨🇭'},
|
||||
'PL': {'name': 'Poland', 'flag': '🇵🇱'},
|
||||
'MX': {'name': 'Mexico', 'flag': '🇲🇽'},
|
||||
'CN': {'name': 'China', 'flag': '🇨🇳'},
|
||||
'KR': {'name': 'South Korea', 'flag': '🇰🇷'},
|
||||
'SG': {'name': 'Singapore', 'flag': '🇸🇬'},
|
||||
'IE': {'name': 'Ireland', 'flag': '🇮🇪'},
|
||||
'NO': {'name': 'Norway', 'flag': '🇳🇴'},
|
||||
'FI': {'name': 'Finland', 'flag': '🇫🇮'},
|
||||
'DK': {'name': 'Denmark', 'flag': '🇩🇰'},
|
||||
'BE': {'name': 'Belgium', 'flag': '🇧🇪'},
|
||||
'AT': {'name': 'Austria', 'flag': '🇦🇹'},
|
||||
'NZ': {'name': 'New Zealand', 'flag': '🇳🇿'},
|
||||
'PT': {'name': 'Portugal', 'flag': '🇵🇹'},
|
||||
'IL': {'name': 'Israel', 'flag': '🇮🇱'},
|
||||
'AR': {'name': 'Argentina', 'flag': '🇦🇷'},
|
||||
'CO': {'name': 'Colombia', 'flag': '🇨🇴'},
|
||||
'CL': {'name': 'Chile', 'flag': '🇨🇱'},
|
||||
'ZA': {'name': 'South Africa', 'flag': '🇿🇦'},
|
||||
'RU': {'name': 'Russia', 'flag': '🇷🇺'},
|
||||
'TR': {'name': 'Turkey', 'flag': '🇹🇷'},
|
||||
'TH': {'name': 'Thailand', 'flag': '🇹🇭'},
|
||||
'MY': {'name': 'Malaysia', 'flag': '🇲🇾'},
|
||||
'ID': {'name': 'Indonesia', 'flag': '🇮🇩'},
|
||||
'PH': {'name': 'Philippines', 'flag': '🇵🇭'},
|
||||
'VN': {'name': 'Vietnam', 'flag': '🇻🇳'},
|
||||
'PK': {'name': 'Pakistan', 'flag': '🇵🇰'},
|
||||
'BD': {'name': 'Bangladesh', 'flag': '🇧🇩'},
|
||||
'UA': {'name': 'Ukraine', 'flag': '🇺🇦'},
|
||||
'RO': {'name': 'Romania', 'flag': '🇷🇴'},
|
||||
'CZ': {'name': 'Czech Republic', 'flag': '🇨🇿'},
|
||||
'GR': {'name': 'Greece', 'flag': '🇬🇷'},
|
||||
'HU': {'name': 'Hungary', 'flag': '🇭🇺'}
|
||||
}
|
||||
|
||||
# Format top countries data
|
||||
top_countries_data = []
|
||||
for country_code, downloads in top_countries:
|
||||
country_data = country_info.get(country_code, {'name': country_code, 'flag': '🌍'})
|
||||
percentage = (downloads / total_all_downloads * 100) if total_all_downloads > 0 else 0
|
||||
|
||||
top_countries_data.append({
|
||||
'code': country_code,
|
||||
'name': country_data['name'],
|
||||
'flag': country_data['flag'],
|
||||
'downloads': downloads,
|
||||
'percentage': round(percentage, 1)
|
||||
})
|
||||
|
||||
print(f"🌍 Top 5 Countries:")
|
||||
for country in top_countries_data:
|
||||
print(f" • {country['flag']} {country['name']}: {country['downloads']:,} ({country['percentage']}%)")
|
||||
|
||||
# Create final structure
|
||||
trending_data = {
|
||||
"lastUpdated": now.isoformat() + "Z",
|
||||
"globalStats": {
|
||||
"totalComponents": total_components,
|
||||
"totalDownloads": total_all_downloads,
|
||||
"monthlyDownloads": total_month_downloads,
|
||||
"weeklyDownloads": total_week_downloads,
|
||||
"todayDownloads": total_today_downloads,
|
||||
"totalCountries": len(unique_countries)
|
||||
},
|
||||
"topCountries": top_countries_data,
|
||||
"trending": {},
|
||||
"chartData": {
|
||||
"dates": chart_dates,
|
||||
"series": chart_series
|
||||
}
|
||||
}
|
||||
|
||||
# Map component types to expected names
|
||||
type_mapping = {
|
||||
'command': 'commands',
|
||||
'commands': 'commands',
|
||||
'agent': 'agents',
|
||||
'agents': 'agents',
|
||||
'setting': 'settings',
|
||||
'settings': 'settings',
|
||||
'hook': 'hooks',
|
||||
'hooks': 'hooks',
|
||||
'mcp': 'mcps',
|
||||
'mcps': 'mcps',
|
||||
'skill': 'skills',
|
||||
'skills': 'skills',
|
||||
'template': 'templates',
|
||||
'templates': 'templates',
|
||||
'plugin': 'plugins',
|
||||
'plugins': 'plugins',
|
||||
'sandbox': 'sandbox'
|
||||
}
|
||||
|
||||
# Debug: Print what component types we found
|
||||
print(f"🔍 Component types found in data: {list(trending_by_type.keys())}")
|
||||
|
||||
# Populate trending data with real data or fallback
|
||||
processed_types = set()
|
||||
for db_type, json_type in type_mapping.items():
|
||||
if json_type not in processed_types and db_type in trending_by_type:
|
||||
trending_data['trending'][json_type] = trending_by_type[db_type]
|
||||
processed_types.add(json_type)
|
||||
print(f"✅ Using real data for {json_type}: {len(trending_by_type[db_type])} items")
|
||||
|
||||
# Add fallback data only for types that don't have real data
|
||||
for json_type in ['commands', 'agents', 'settings', 'hooks', 'mcps', 'skills', 'templates']:
|
||||
if json_type not in trending_data['trending']:
|
||||
trending_data['trending'][json_type] = create_fallback_data(json_type)
|
||||
print(f"⚠️ Using fallback data for {json_type}")
|
||||
|
||||
# Add "all" category with top 10 across all categories
|
||||
all_items = []
|
||||
for items in trending_by_type.values():
|
||||
all_items.extend(items)
|
||||
|
||||
# Sort all items by weekly downloads and take top 10
|
||||
all_items.sort(key=lambda x: x['downloadsWeek'], reverse=True)
|
||||
trending_data['trending']['all'] = all_items[:10]
|
||||
|
||||
return trending_data
|
||||
|
||||
def create_fallback_data(component_type):
|
||||
"""Create fallback data for component types with no real data"""
|
||||
|
||||
fallback_data = {
|
||||
'commands': [
|
||||
{
|
||||
'id': 'react-component-generator',
|
||||
'name': 'React Component Generator',
|
||||
'category': 'frontend',
|
||||
'downloadsToday': 45,
|
||||
'downloadsWeek': 234,
|
||||
'downloadsMonth': 567,
|
||||
'downloadsTotal': 1248
|
||||
},
|
||||
{
|
||||
'id': 'api-endpoint-generator',
|
||||
'name': 'API Endpoint Generator',
|
||||
'category': 'backend',
|
||||
'downloadsToday': 32,
|
||||
'downloadsWeek': 189,
|
||||
'downloadsMonth': 445,
|
||||
'downloadsTotal': 967
|
||||
}
|
||||
],
|
||||
'agents': [
|
||||
{
|
||||
'id': 'react-expert',
|
||||
'name': 'React Performance Expert',
|
||||
'category': 'frontend',
|
||||
'downloadsToday': 28,
|
||||
'downloadsWeek': 156,
|
||||
'downloadsMonth': 389,
|
||||
'downloadsTotal': 834
|
||||
},
|
||||
{
|
||||
'id': 'security-analyst',
|
||||
'name': 'Security Code Analyst',
|
||||
'category': 'security',
|
||||
'downloadsToday': 35,
|
||||
'downloadsWeek': 178,
|
||||
'downloadsMonth': 423,
|
||||
'downloadsTotal': 912
|
||||
}
|
||||
],
|
||||
'settings': [
|
||||
{
|
||||
'id': 'vscode-theme',
|
||||
'name': 'Optimized VSCode Settings',
|
||||
'category': 'editor',
|
||||
'downloadsToday': 67,
|
||||
'downloadsWeek': 345,
|
||||
'downloadsMonth': 892,
|
||||
'downloadsTotal': 1876
|
||||
}
|
||||
],
|
||||
'hooks': [
|
||||
{
|
||||
'id': 'pre-commit-tests',
|
||||
'name': 'Pre-commit Test Runner',
|
||||
'category': 'testing',
|
||||
'downloadsToday': 23,
|
||||
'downloadsWeek': 123,
|
||||
'downloadsMonth': 298,
|
||||
'downloadsTotal': 645
|
||||
}
|
||||
],
|
||||
'mcps': [
|
||||
{
|
||||
'id': 'github-integration',
|
||||
'name': 'GitHub API Integration',
|
||||
'category': 'git',
|
||||
'downloadsToday': 41,
|
||||
'downloadsWeek': 198,
|
||||
'downloadsMonth': 456,
|
||||
'downloadsTotal': 1023
|
||||
}
|
||||
],
|
||||
'templates': [
|
||||
{
|
||||
'id': 'nextjs-starter',
|
||||
'name': 'Next.js Starter Template',
|
||||
'category': 'frontend',
|
||||
'downloadsToday': 52,
|
||||
'downloadsWeek': 267,
|
||||
'downloadsMonth': 634,
|
||||
'downloadsTotal': 1387
|
||||
}
|
||||
],
|
||||
'skills': [
|
||||
{
|
||||
'id': 'data-visualization',
|
||||
'name': 'Data Visualization Expert',
|
||||
'category': 'data-science',
|
||||
'downloadsToday': 38,
|
||||
'downloadsWeek': 201,
|
||||
'downloadsMonth': 512,
|
||||
'downloadsTotal': 1129
|
||||
},
|
||||
{
|
||||
'id': 'api-documentation',
|
||||
'name': 'API Documentation Generator',
|
||||
'category': 'documentation',
|
||||
'downloadsToday': 29,
|
||||
'downloadsWeek': 167,
|
||||
'downloadsMonth': 423,
|
||||
'downloadsTotal': 934
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
return fallback_data.get(component_type, [])
|
||||
|
||||
def generate_fallback_trending_data():
|
||||
"""Generate complete fallback trending data structure"""
|
||||
return {
|
||||
"lastUpdated": datetime.now().isoformat() + "Z",
|
||||
"trending": {
|
||||
"commands": [
|
||||
{
|
||||
'id': 'react-component-generator',
|
||||
'name': 'React Component Generator',
|
||||
'category': 'frontend',
|
||||
'downloadsToday': 127,
|
||||
'downloadsWeek': 892,
|
||||
'downloadsMonth': 2847,
|
||||
'downloadsTotal': 5634
|
||||
},
|
||||
{
|
||||
'id': 'api-endpoint-generator',
|
||||
'name': 'API Endpoint Generator',
|
||||
'category': 'backend',
|
||||
'downloadsToday': 74,
|
||||
'downloadsWeek': 389,
|
||||
'downloadsMonth': 1089,
|
||||
'downloadsTotal': 2834
|
||||
},
|
||||
{
|
||||
'id': 'database-migration-system',
|
||||
'name': 'Database Migration System',
|
||||
'category': 'database',
|
||||
'downloadsToday': 45,
|
||||
'downloadsWeek': 234,
|
||||
'downloadsMonth': 567,
|
||||
'downloadsTotal': 1432
|
||||
},
|
||||
{
|
||||
'id': 'docker-setup-wizard',
|
||||
'name': 'Docker Setup Wizard',
|
||||
'category': 'devops',
|
||||
'downloadsToday': 89,
|
||||
'downloadsWeek': 445,
|
||||
'downloadsMonth': 1234,
|
||||
'downloadsTotal': 2967
|
||||
},
|
||||
{
|
||||
'id': 'unit-test-generator',
|
||||
'name': 'Unit Test Generator',
|
||||
'category': 'testing',
|
||||
'downloadsToday': 56,
|
||||
'downloadsWeek': 298,
|
||||
'downloadsMonth': 789,
|
||||
'downloadsTotal': 1876
|
||||
}
|
||||
],
|
||||
"agents": [
|
||||
{
|
||||
'id': 'react-expert',
|
||||
'name': 'React Performance Expert',
|
||||
'category': 'frontend',
|
||||
'downloadsToday': 98,
|
||||
'downloadsWeek': 567,
|
||||
'downloadsMonth': 1456,
|
||||
'downloadsTotal': 3245
|
||||
},
|
||||
{
|
||||
'id': 'security-analyst',
|
||||
'name': 'Security Code Analyst',
|
||||
'category': 'security',
|
||||
'downloadsToday': 112,
|
||||
'downloadsWeek': 634,
|
||||
'downloadsMonth': 1789,
|
||||
'downloadsTotal': 4123
|
||||
},
|
||||
{
|
||||
'id': 'api-architect',
|
||||
'name': 'API Architecture Specialist',
|
||||
'category': 'backend',
|
||||
'downloadsToday': 67,
|
||||
'downloadsWeek': 345,
|
||||
'downloadsMonth': 923,
|
||||
'downloadsTotal': 2456
|
||||
},
|
||||
{
|
||||
'id': 'database-optimizer',
|
||||
'name': 'Database Performance Optimizer',
|
||||
'category': 'database',
|
||||
'downloadsToday': 43,
|
||||
'downloadsWeek': 198,
|
||||
'downloadsMonth': 534,
|
||||
'downloadsTotal': 1234
|
||||
}
|
||||
],
|
||||
"settings": [
|
||||
{
|
||||
'id': 'vscode-theme',
|
||||
'name': 'Optimized VSCode Settings',
|
||||
'category': 'editor',
|
||||
'downloadsToday': 234,
|
||||
'downloadsWeek': 1234,
|
||||
'downloadsMonth': 3456,
|
||||
'downloadsTotal': 7891
|
||||
},
|
||||
{
|
||||
'id': 'eslint-config',
|
||||
'name': 'Strict ESLint Configuration',
|
||||
'category': 'linting',
|
||||
'downloadsToday': 156,
|
||||
'downloadsWeek': 789,
|
||||
'downloadsMonth': 2134,
|
||||
'downloadsTotal': 4567
|
||||
},
|
||||
{
|
||||
'id': 'prettier-setup',
|
||||
'name': 'Team Prettier Standards',
|
||||
'category': 'formatting',
|
||||
'downloadsToday': 134,
|
||||
'downloadsWeek': 678,
|
||||
'downloadsMonth': 1876,
|
||||
'downloadsTotal': 3892
|
||||
}
|
||||
],
|
||||
"hooks": [
|
||||
{
|
||||
'id': 'pre-commit-tests',
|
||||
'name': 'Pre-commit Test Runner',
|
||||
'category': 'testing',
|
||||
'downloadsToday': 87,
|
||||
'downloadsWeek': 456,
|
||||
'downloadsMonth': 1123,
|
||||
'downloadsTotal': 2789
|
||||
},
|
||||
{
|
||||
'id': 'code-formatter',
|
||||
'name': 'Auto Code Formatter',
|
||||
'category': 'formatting',
|
||||
'downloadsToday': 76,
|
||||
'downloadsWeek': 389,
|
||||
'downloadsMonth': 934,
|
||||
'downloadsTotal': 2134
|
||||
}
|
||||
],
|
||||
"mcps": [
|
||||
{
|
||||
'id': 'github-integration',
|
||||
'name': 'GitHub API Integration',
|
||||
'category': 'git',
|
||||
'downloadsToday': 145,
|
||||
'downloadsWeek': 723,
|
||||
'downloadsMonth': 1987,
|
||||
'downloadsTotal': 4321
|
||||
},
|
||||
{
|
||||
'id': 'slack-notifications',
|
||||
'name': 'Slack Notification System',
|
||||
'category': 'communication',
|
||||
'downloadsToday': 98,
|
||||
'downloadsWeek': 456,
|
||||
'downloadsMonth': 1234,
|
||||
'downloadsTotal': 2987
|
||||
}
|
||||
],
|
||||
"templates": [
|
||||
{
|
||||
'id': 'nextjs-starter',
|
||||
'name': 'Next.js Starter Template',
|
||||
'category': 'frontend',
|
||||
'downloadsToday': 189,
|
||||
'downloadsWeek': 945,
|
||||
'downloadsMonth': 2567,
|
||||
'downloadsTotal': 5432
|
||||
},
|
||||
{
|
||||
'id': 'express-api',
|
||||
'name': 'Express API Template',
|
||||
'category': 'backend',
|
||||
'downloadsToday': 123,
|
||||
'downloadsWeek': 612,
|
||||
'downloadsMonth': 1678,
|
||||
'downloadsTotal': 3456
|
||||
}
|
||||
],
|
||||
"skills": [
|
||||
{
|
||||
'id': 'data-visualization',
|
||||
'name': 'Data Visualization Expert',
|
||||
'category': 'data-science',
|
||||
'downloadsToday': 87,
|
||||
'downloadsWeek': 478,
|
||||
'downloadsMonth': 1289,
|
||||
'downloadsTotal': 2834
|
||||
},
|
||||
{
|
||||
'id': 'api-documentation',
|
||||
'name': 'API Documentation Generator',
|
||||
'category': 'documentation',
|
||||
'downloadsToday': 64,
|
||||
'downloadsWeek': 356,
|
||||
'downloadsMonth': 923,
|
||||
'downloadsTotal': 2145
|
||||
},
|
||||
{
|
||||
'id': 'code-review',
|
||||
'name': 'Intelligent Code Reviewer',
|
||||
'category': 'quality-assurance',
|
||||
'downloadsToday': 101,
|
||||
'downloadsWeek': 534,
|
||||
'downloadsMonth': 1456,
|
||||
'downloadsTotal': 3127
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+231
@@ -0,0 +1,231 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Pre-Deployment Validation Script
|
||||
# This script MUST pass before deploying to production
|
||||
|
||||
set -e # Exit on any error
|
||||
|
||||
echo "🚀 Pre-Deployment Validation Check"
|
||||
echo "=================================="
|
||||
echo ""
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Counter
|
||||
TESTS_PASSED=0
|
||||
TESTS_FAILED=0
|
||||
|
||||
# Function to print success
|
||||
success() {
|
||||
echo -e "${GREEN}✓${NC} $1"
|
||||
((TESTS_PASSED++))
|
||||
}
|
||||
|
||||
# Function to print error
|
||||
error() {
|
||||
echo -e "${RED}✗${NC} $1"
|
||||
((TESTS_FAILED++))
|
||||
}
|
||||
|
||||
# Function to print warning
|
||||
warning() {
|
||||
echo -e "${YELLOW}⚠${NC} $1"
|
||||
}
|
||||
|
||||
# Function to print info
|
||||
info() {
|
||||
echo -e "${BLUE}ℹ${NC} $1"
|
||||
}
|
||||
|
||||
echo "Step 1: Checking Git Status"
|
||||
echo "----------------------------"
|
||||
|
||||
# Check if we're in a git repository
|
||||
if ! git rev-parse --git-dir > /dev/null 2>&1; then
|
||||
error "Not a git repository"
|
||||
exit 1
|
||||
else
|
||||
success "Git repository detected"
|
||||
fi
|
||||
|
||||
# Check for uncommitted changes
|
||||
if [[ -n $(git status -s) ]]; then
|
||||
warning "There are uncommitted changes"
|
||||
git status -s
|
||||
echo ""
|
||||
read -p "Continue anyway? (y/N) " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
success "No uncommitted changes"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 2: Checking Node.js & npm"
|
||||
echo "-------------------------------"
|
||||
|
||||
# Check Node.js version
|
||||
if command -v node &> /dev/null; then
|
||||
NODE_VERSION=$(node --version)
|
||||
success "Node.js ${NODE_VERSION} installed"
|
||||
else
|
||||
error "Node.js not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check npm
|
||||
if command -v npm &> /dev/null; then
|
||||
NPM_VERSION=$(npm --version)
|
||||
success "npm ${NPM_VERSION} installed"
|
||||
else
|
||||
error "npm not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 3: Installing/Updating Dependencies"
|
||||
echo "-----------------------------------------"
|
||||
|
||||
# Install root dependencies
|
||||
if [ -f "package.json" ]; then
|
||||
info "Installing root dependencies..."
|
||||
if npm install --silent --ignore-scripts; then
|
||||
success "Root dependencies installed"
|
||||
else
|
||||
error "Failed to install root dependencies"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Install API dependencies
|
||||
if [ -f "api/package.json" ]; then
|
||||
info "Installing API dependencies..."
|
||||
cd api
|
||||
if npm install --silent --ignore-scripts; then
|
||||
success "API dependencies installed"
|
||||
else
|
||||
error "Failed to install API dependencies"
|
||||
exit 1
|
||||
fi
|
||||
cd ..
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 4: Running API Tests"
|
||||
echo "-------------------------"
|
||||
|
||||
cd api
|
||||
|
||||
# Run critical API tests
|
||||
info "Running critical endpoint tests..."
|
||||
if npm run test:api --silent; then
|
||||
success "All API tests passed"
|
||||
else
|
||||
error "API tests failed"
|
||||
echo ""
|
||||
echo "Fix the failing tests before deploying!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd ..
|
||||
|
||||
echo ""
|
||||
echo "Step 5: Checking API File Structure"
|
||||
echo "------------------------------------"
|
||||
|
||||
# Check critical API endpoints exist
|
||||
CRITICAL_ENDPOINTS=(
|
||||
"api/track-download-supabase.js"
|
||||
"api/discord/interactions.js"
|
||||
"api/claude-code-check.js"
|
||||
)
|
||||
|
||||
for endpoint in "${CRITICAL_ENDPOINTS[@]}"; do
|
||||
if [ -f "$endpoint" ]; then
|
||||
success "$(basename $endpoint) exists"
|
||||
else
|
||||
error "$(basename $endpoint) NOT FOUND - Critical endpoint missing!"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Step 6: Validating Environment Variables"
|
||||
echo "-----------------------------------------"
|
||||
|
||||
# Check if .env.example exists
|
||||
if [ -f ".env.example" ]; then
|
||||
success ".env.example file exists"
|
||||
|
||||
# Extract required variables
|
||||
REQUIRED_VARS=$(grep -E "^[A-Z_]+=" .env.example | cut -d'=' -f1 | sort | uniq)
|
||||
|
||||
info "Required environment variables:"
|
||||
echo "$REQUIRED_VARS" | while read var; do
|
||||
echo " - $var"
|
||||
done
|
||||
echo ""
|
||||
warning "Make sure these are set in Vercel Dashboard"
|
||||
else
|
||||
warning ".env.example not found"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 7: Vercel Configuration Check"
|
||||
echo "-----------------------------------"
|
||||
|
||||
# Check vercel.json
|
||||
if [ -f "vercel.json" ]; then
|
||||
success "vercel.json exists"
|
||||
|
||||
# Validate it's valid JSON
|
||||
if jq empty vercel.json 2>/dev/null; then
|
||||
success "vercel.json is valid JSON"
|
||||
else
|
||||
error "vercel.json is invalid JSON"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
warning "vercel.json not found"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 8: Final Checks"
|
||||
echo "--------------------"
|
||||
|
||||
# Check if vercel CLI is installed
|
||||
if command -v vercel &> /dev/null; then
|
||||
VERCEL_VERSION=$(vercel --version)
|
||||
success "Vercel CLI ${VERCEL_VERSION} installed"
|
||||
else
|
||||
warning "Vercel CLI not installed (install with: npm i -g vercel)"
|
||||
fi
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo "=================================="
|
||||
echo "Pre-Deployment Check Summary"
|
||||
echo "=================================="
|
||||
echo -e "${GREEN}Passed: ${TESTS_PASSED}${NC}"
|
||||
echo -e "${RED}Failed: ${TESTS_FAILED}${NC}"
|
||||
echo ""
|
||||
|
||||
if [ $TESTS_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All checks passed! Ready to deploy.${NC}"
|
||||
echo ""
|
||||
echo "Deploy with:"
|
||||
echo " vercel --prod"
|
||||
echo ""
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}✗ Some checks failed. Fix issues before deploying.${NC}"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
Executable
+323
@@ -0,0 +1,323 @@
|
||||
#!/usr/bin/env bash
|
||||
# run-review-cycle.sh
|
||||
# Manual trigger for a component review cycle.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/run-review-cycle.sh # full cycle, auto-select component
|
||||
# ./scripts/run-review-cycle.sh --component PATH # full cycle, specific component
|
||||
# ./scripts/run-review-cycle.sh --test-hook # test hook logging only (fast, ~10s)
|
||||
# ./scripts/run-review-cycle.sh --check # check current state (no changes)
|
||||
#
|
||||
# Modes:
|
||||
# --test-hook Creates a real cycle, writes marker, fires mock tool events through
|
||||
# the hook, then reads back from the API to confirm tools were logged.
|
||||
# This is the fastest way to verify the fix works.
|
||||
#
|
||||
# --check Just shows current cycles + control state from the API.
|
||||
#
|
||||
# (default) Full cycle: creates cycle, writes marker, runs Claude with the
|
||||
# component-review skill, monitors until done.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── constants ──────────────────────────────────────────────────────────────────
|
||||
API_BASE="https://www.aitmpl.com/api/live-task"
|
||||
MARKER="/tmp/.claude-review-active"
|
||||
REPO="/Users/danipower/Proyectos/Github/claude-code-templates"
|
||||
HOOK_GLOBAL="/Users/danipower/.claude/hooks/live-task-tool-logger.sh"
|
||||
HOOK_PROJECT="$REPO/.claude/hooks/log-review-tools.sh"
|
||||
SKILL_PATH="/Users/danipower/.claude/scheduled-tasks/component-review/SKILL.md"
|
||||
|
||||
# ── colors ─────────────────────────────────────────────────────────────────────
|
||||
R='\033[0;31m'; G='\033[0;32m'; Y='\033[1;33m'; B='\033[0;34m'; C='\033[0;36m'; N='\033[0m'
|
||||
ok() { echo -e "${G}✓${N} $*"; }
|
||||
info() { echo -e "${B}→${N} $*"; }
|
||||
warn() { echo -e "${Y}⚠${N} $*"; }
|
||||
err() { echo -e "${R}✗${N} $*" >&2; }
|
||||
head() { echo -e "\n${C}── $* ──${N}"; }
|
||||
|
||||
# ── argument parsing ───────────────────────────────────────────────────────────
|
||||
MODE="full"
|
||||
COMPONENT_PATH=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--test-hook) MODE="test-hook"; shift ;;
|
||||
--check) MODE="check"; shift ;;
|
||||
--component) COMPONENT_PATH="$2"; shift 2 ;;
|
||||
-h|--help)
|
||||
sed -n '3,20p' "$0" | sed 's/^# \?//'
|
||||
exit 0 ;;
|
||||
*)
|
||||
err "Unknown arg: $1 (use --help)"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── prereqs ────────────────────────────────────────────────────────────────────
|
||||
for cmd in curl jq; do
|
||||
command -v "$cmd" >/dev/null || { err "Missing: $cmd"; exit 1; }
|
||||
done
|
||||
|
||||
cleanup() {
|
||||
rm -f "$MARKER"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# MODE: --check
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
if [[ "$MODE" == "check" ]]; then
|
||||
head "Control state"
|
||||
curl -s "$API_BASE/control" | jq .
|
||||
|
||||
head "Last 10 cycles"
|
||||
curl -s "$API_BASE/cycles?limit=10" | jq -r '
|
||||
.cycles[] |
|
||||
"\(.id) \(.status)\t\(.phase)\t\(.component_path | split("/")[-1])\t\(.started_at[:16])"
|
||||
' | column -t
|
||||
|
||||
if [[ -f "$MARKER" ]]; then
|
||||
head "Active marker file"
|
||||
cat "$MARKER"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── shared helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
check_paused() {
|
||||
local data
|
||||
data=$(curl -s "$API_BASE/control")
|
||||
if echo "$data" | jq -e '.control.is_paused == true' >/dev/null 2>&1; then
|
||||
warn "Review loop is PAUSED. Resume from the dashboard before running."
|
||||
exit 1
|
||||
fi
|
||||
ok "Loop is running (not paused)"
|
||||
}
|
||||
|
||||
auto_select_component() {
|
||||
# Pick a component that hasn't been reviewed recently
|
||||
local recent
|
||||
recent=$(curl -s "$API_BASE/cycles?limit=20" | jq -r '[.cycles[].component_path] | @json')
|
||||
|
||||
local chosen=""
|
||||
while IFS= read -r -d '' path; do
|
||||
rel="${path#$REPO/}"
|
||||
if ! echo "$recent" | jq -e --arg p "$rel" '. | index($p) != null' >/dev/null 2>&1; then
|
||||
chosen="$rel"
|
||||
break
|
||||
fi
|
||||
done < <(find "$REPO/cli-tool/components/agents" -name "*.md" -print0 | sort -z)
|
||||
|
||||
if [[ -z "$chosen" ]]; then
|
||||
# Fallback: pick first agent
|
||||
chosen=$(find "$REPO/cli-tool/components/agents" -name "*.md" | head -1)
|
||||
chosen="${chosen#$REPO/}"
|
||||
fi
|
||||
|
||||
echo "$chosen"
|
||||
}
|
||||
|
||||
create_cycle() {
|
||||
local component="$1" session_id="$2"
|
||||
local type
|
||||
type=$(echo "$component" | awk -F/ '{print $3}') # agents / commands / mcps …
|
||||
|
||||
local resp
|
||||
resp=$(curl -s -X POST "$API_BASE/cycles" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"session_id\":\"$session_id\",\"component_path\":\"$component\",\"component_type\":\"$type\"}")
|
||||
|
||||
if echo "$resp" | jq -e '.error' >/dev/null 2>&1; then
|
||||
err "create_cycle failed: $(echo "$resp" | jq -r '.error')"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$resp" | jq -r '.cycle.id'
|
||||
}
|
||||
|
||||
write_marker() {
|
||||
local session_id="$1" cycle_id="$2" phase="$3"
|
||||
printf 'session_id=%s\ncycle_id=%s\nphase=%s\n' "$session_id" "$cycle_id" "$phase" > "$MARKER"
|
||||
}
|
||||
|
||||
update_cycle() {
|
||||
local cycle_id="$1"; shift
|
||||
curl -s -X PATCH "$API_BASE/cycles" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"id\":$cycle_id,$*}" >/dev/null
|
||||
}
|
||||
|
||||
count_tools() {
|
||||
local cycle_id="$1"
|
||||
curl -s "$API_BASE/tools?cycle_id=$cycle_id" | jq '.tools | length'
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# MODE: --test-hook
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
if [[ "$MODE" == "test-hook" ]]; then
|
||||
head "Pre-flight"
|
||||
check_paused
|
||||
|
||||
if [[ ! -x "$HOOK_GLOBAL" ]]; then
|
||||
err "Global hook not found or not executable: $HOOK_GLOBAL"
|
||||
exit 1
|
||||
fi
|
||||
ok "Global hook exists"
|
||||
|
||||
head "Creating test cycle"
|
||||
SESSION_ID="test-hook-$(date +%Y%m%d-%H%M%S)"
|
||||
COMPONENT_PATH="${COMPONENT_PATH:-cli-tool/components/agents/development-tools/debugger.md}"
|
||||
CYCLE_ID=$(create_cycle "$COMPONENT_PATH" "$SESSION_ID")
|
||||
ok "Cycle created: id=$CYCLE_ID component=$(basename $COMPONENT_PATH)"
|
||||
|
||||
head "Writing marker (key=value format)"
|
||||
write_marker "$SESSION_ID" "$CYCLE_ID" "test"
|
||||
cat "$MARKER"
|
||||
ok "Marker written to $MARKER"
|
||||
|
||||
head "Firing mock tool events through global hook"
|
||||
TOOL_NAMES=("Read" "Bash" "Grep" "WebFetch" "Edit")
|
||||
FIRED=0
|
||||
for tool in "${TOOL_NAMES[@]}"; do
|
||||
mock_input=$(jq -nc \
|
||||
--arg tool "$tool" \
|
||||
--arg sid "any-session-value" \
|
||||
'{tool_name: $tool, session_id: $sid, tool_input: {file_path: "/tmp/test.md"}, tool_response: "ok"}')
|
||||
|
||||
echo "$mock_input" | bash "$HOOK_GLOBAL"
|
||||
sleep 0.3
|
||||
FIRED=$((FIRED + 1))
|
||||
echo -e " ${G}·${N} $tool fired"
|
||||
done
|
||||
|
||||
head "Waiting for async POSTs to land (2s)"
|
||||
sleep 2
|
||||
|
||||
head "Verifying tool executions in API"
|
||||
TOOL_COUNT=$(count_tools "$CYCLE_ID")
|
||||
if [[ "$TOOL_COUNT" -ge "$FIRED" ]]; then
|
||||
ok "$TOOL_COUNT tool(s) logged for cycle $CYCLE_ID — hook is working correctly!"
|
||||
elif [[ "$TOOL_COUNT" -gt 0 ]]; then
|
||||
warn "$TOOL_COUNT / $FIRED tools logged (some may still be in-flight)"
|
||||
else
|
||||
err "0 tools logged — hook is NOT working. Check $HOOK_GLOBAL"
|
||||
echo ""
|
||||
echo "Debug: try running the hook manually:"
|
||||
echo " echo '{\"tool_name\":\"Read\",\"session_id\":\"x\",\"tool_input\":{},\"tool_response\":\"ok\"}' | bash $HOOK_GLOBAL"
|
||||
fi
|
||||
|
||||
# Show the logged tools
|
||||
if [[ "$TOOL_COUNT" -gt 0 ]]; then
|
||||
curl -s "$API_BASE/tools?cycle_id=$CYCLE_ID" | jq -r '
|
||||
.tools[] | " [\(.result_status)] \(.tool_name) \(.tool_args_summary // "")"
|
||||
'
|
||||
fi
|
||||
|
||||
head "Cleaning up test cycle"
|
||||
update_cycle "$CYCLE_ID" '"status":"failed","error_message":"test-hook run — auto-cleaned"'
|
||||
rm -f "$MARKER"
|
||||
ok "Cycle $CYCLE_ID marked failed + marker removed"
|
||||
|
||||
echo ""
|
||||
echo -e "${G}Test complete.${N} Check https://www.aitmpl.com/live-task (filter: All) to see the test cycle."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# MODE: full cycle
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
head "Pre-flight"
|
||||
check_paused
|
||||
|
||||
# Check for already-active cycles
|
||||
ACTIVE=$(curl -s "$API_BASE/cycles?status=active&limit=1" | jq '.cycles | length')
|
||||
if [[ "$ACTIVE" -gt 0 ]]; then
|
||||
warn "An active cycle already exists. Use --check to inspect it."
|
||||
echo -n "Continue anyway? [y/N] "
|
||||
read -r ans
|
||||
[[ "${ans,,}" == "y" ]] || exit 0
|
||||
fi
|
||||
|
||||
head "Component selection"
|
||||
if [[ -z "$COMPONENT_PATH" ]]; then
|
||||
COMPONENT_PATH=$(auto_select_component)
|
||||
info "Auto-selected: $COMPONENT_PATH"
|
||||
else
|
||||
info "Using: $COMPONENT_PATH"
|
||||
fi
|
||||
|
||||
if [[ ! -f "$REPO/$COMPONENT_PATH" ]]; then
|
||||
err "Component file not found: $REPO/$COMPONENT_PATH"
|
||||
exit 1
|
||||
fi
|
||||
ok "Component file exists"
|
||||
|
||||
head "Creating cycle"
|
||||
SESSION_ID="review-$(date +%Y%m%d-%H%M%S)"
|
||||
CYCLE_ID=$(create_cycle "$COMPONENT_PATH" "$SESSION_ID")
|
||||
ok "Cycle created: id=$CYCLE_ID session=$SESSION_ID"
|
||||
|
||||
head "Writing marker"
|
||||
write_marker "$SESSION_ID" "$CYCLE_ID" "research"
|
||||
ok "Marker written: $(cat $MARKER | tr '\n' ' ')"
|
||||
|
||||
head "Updating phase → research"
|
||||
update_cycle "$CYCLE_ID" '"phase":"research"'
|
||||
ok "Phase set"
|
||||
|
||||
echo ""
|
||||
echo -e "${C}═══════════════════════════════════════════════════════${N}"
|
||||
echo -e "${C} Cycle $CYCLE_ID ready. Starting Claude review...${N}"
|
||||
echo -e "${C} Dashboard: https://www.aitmpl.com/live-task${N}"
|
||||
echo -e "${C}═══════════════════════════════════════════════════════${N}"
|
||||
echo ""
|
||||
|
||||
# Read the skill prompt
|
||||
SKILL_PROMPT=$(cat "$SKILL_PATH")
|
||||
|
||||
# Inject the selected component so Claude doesn't have to auto-select
|
||||
INJECTED_PROMPT="$(cat <<EOF
|
||||
You are the Component Review Loop orchestrator. A cycle has already been created for you:
|
||||
|
||||
- cycle_id: $CYCLE_ID
|
||||
- session_id: $SESSION_ID
|
||||
- component_path: $COMPONENT_PATH
|
||||
- component_type: $(echo "$COMPONENT_PATH" | awk -F/ '{print $3}')
|
||||
- marker file already written at /tmp/.claude-review-active
|
||||
|
||||
**Skip steps 1–6** (pause check, stuck cycle check, component selection, session id, cycle creation, marker write — all done).
|
||||
|
||||
**Start from step 7** (set phase to research) and continue through to completion.
|
||||
|
||||
Follow the SKILL instructions below:
|
||||
|
||||
---
|
||||
|
||||
$SKILL_PROMPT
|
||||
EOF
|
||||
)"
|
||||
|
||||
# Run Claude non-interactively with the skill prompt
|
||||
claude \
|
||||
--dangerously-skip-permissions \
|
||||
--print \
|
||||
--add-dir "$REPO" \
|
||||
--output-format text \
|
||||
-p "$INJECTED_PROMPT" 2>&1
|
||||
|
||||
EXIT_CODE=$?
|
||||
|
||||
echo ""
|
||||
head "Post-run check"
|
||||
TOOL_COUNT=$(count_tools "$CYCLE_ID")
|
||||
ok "$TOOL_COUNT tool(s) logged for cycle $CYCLE_ID"
|
||||
|
||||
FINAL_STATUS=$(curl -s "$API_BASE/cycles?limit=20" | jq -r --argjson id "$CYCLE_ID" '.cycles[] | select(.id == ($id | tostring) or .id == $id) | .status')
|
||||
info "Cycle status: ${FINAL_STATUS:-unknown}"
|
||||
|
||||
echo ""
|
||||
echo "Full details: https://www.aitmpl.com/live-task"
|
||||
|
||||
exit $EXIT_CODE
|
||||
@@ -0,0 +1,401 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Batch-orchestrate SkillSpector scans over skill components.
|
||||
|
||||
Discovers skill directories under cli-tool/components/skills/, runs the
|
||||
SkillSpector CLI (`skillspector scan <dir> --no-llm --format json`) on each,
|
||||
aggregates the per-skill JSON reports, and emits:
|
||||
|
||||
* a Markdown summary (for PR comments / step summaries)
|
||||
* an aggregated SARIF 2.1.0 log (for GitHub code scanning)
|
||||
* GitHub Action outputs (failed, high_critical_count, scanned, errors)
|
||||
|
||||
SkillSpector itself requires Python 3.12+, but this orchestrator only uses the
|
||||
standard library and runs on 3.9+ so the discovery logic can be exercised
|
||||
locally without installing the scanner (see --dry-run).
|
||||
|
||||
Usage:
|
||||
skillspector_scan.py --all
|
||||
skillspector_scan.py --changed --base-ref origin/main
|
||||
skillspector_scan.py --all --dry-run # list dirs, no scanning
|
||||
|
||||
The CLI exit code of `skillspector scan` is intentionally ignored: it returns
|
||||
1 when a skill's risk score is high (a signal, not a failure) and 2 on real
|
||||
errors. We parse stdout JSON and classify per-skill instead, so one broken
|
||||
skill never aborts the whole batch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Repo-relative root that holds every skill component.
|
||||
SKILLS_ROOT = Path("cli-tool/components/skills")
|
||||
|
||||
# Accepted manifest filenames (case variants seen in the catalog).
|
||||
MANIFEST_NAMES = {"skill.md"}
|
||||
|
||||
# Risk score above which a skill is considered HIGH/CRITICAL.
|
||||
DEFAULT_THRESHOLD = 50
|
||||
|
||||
# Keep PR comments under GitHub's ~65k character limit.
|
||||
MAX_COMMENT_CHARS = 60000
|
||||
|
||||
# Marker so the PR comment can be found and updated idempotently.
|
||||
COMMENT_MARKER = "<!-- skillspector-report:v1 -->"
|
||||
|
||||
|
||||
def _has_manifest(directory: Path) -> bool:
|
||||
"""Return True if directory directly contains a SKILL.md (any case)."""
|
||||
try:
|
||||
for entry in directory.iterdir():
|
||||
if entry.is_file() and entry.name.lower() in MANIFEST_NAMES:
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def discover_all_skills(root: Path = SKILLS_ROOT) -> list[Path]:
|
||||
"""Find every skill directory (one that directly contains a SKILL.md)."""
|
||||
if not root.is_dir():
|
||||
return []
|
||||
skills: list[Path] = []
|
||||
for dirpath, _dirnames, filenames in os.walk(root):
|
||||
if any(name.lower() in MANIFEST_NAMES for name in filenames):
|
||||
skills.append(Path(dirpath))
|
||||
return sorted(set(skills))
|
||||
|
||||
|
||||
def _changed_files(base_ref: str) -> list[Path]:
|
||||
"""Return repo-relative paths changed vs base_ref (three-dot diff)."""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["git", "diff", "--name-only", f"{base_ref}...HEAD"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout
|
||||
except subprocess.CalledProcessError:
|
||||
# Fall back to two-dot if the merge base is unavailable (shallow clone).
|
||||
out = subprocess.run(
|
||||
["git", "diff", "--name-only", base_ref],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
).stdout
|
||||
return [Path(line.strip()) for line in out.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def discover_changed_skills(base_ref: str, root: Path = SKILLS_ROOT) -> list[Path]:
|
||||
"""Map changed files under the skills tree to their owning skill dirs."""
|
||||
changed = _changed_files(base_ref)
|
||||
skills: set[Path] = set()
|
||||
root_resolved = root.resolve()
|
||||
for rel in changed:
|
||||
# Only consider files under the skills tree.
|
||||
try:
|
||||
rel.resolve().relative_to(root_resolved)
|
||||
except ValueError:
|
||||
if root not in rel.parents and rel != root:
|
||||
continue
|
||||
# Walk up from the file's directory until we find the skill dir.
|
||||
current = (rel if rel.is_dir() else rel.parent)
|
||||
while True:
|
||||
if current == root or current == Path("."):
|
||||
break
|
||||
if current.is_dir() and _has_manifest(current):
|
||||
skills.add(current)
|
||||
break
|
||||
# Stop once we climb above the skills root.
|
||||
if root not in current.parents:
|
||||
break
|
||||
current = current.parent
|
||||
return sorted(skills)
|
||||
|
||||
|
||||
def scan_skill(directory: Path) -> dict:
|
||||
"""Run skillspector on one skill dir; return a normalized result dict."""
|
||||
result: dict = {"path": str(directory), "status": "ok"}
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
"skillspector",
|
||||
"scan",
|
||||
str(directory),
|
||||
"--no-llm",
|
||||
"--format",
|
||||
"json",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
result["status"] = "error"
|
||||
result["error"] = "skillspector CLI not found on PATH"
|
||||
return result
|
||||
|
||||
# Exit code 2 == real error. 0/1 are both valid (1 == high score).
|
||||
if proc.returncode == 2:
|
||||
result["status"] = "error"
|
||||
result["error"] = (proc.stderr or proc.stdout or "scan failed").strip()[:500]
|
||||
return result
|
||||
|
||||
try:
|
||||
data = json.loads(proc.stdout)
|
||||
except json.JSONDecodeError:
|
||||
result["status"] = "error"
|
||||
result["error"] = "could not parse skillspector JSON output"
|
||||
result["raw"] = (proc.stdout or proc.stderr or "")[:500]
|
||||
return result
|
||||
|
||||
risk = data.get("risk_assessment", {})
|
||||
result["name"] = (data.get("skill") or {}).get("name") or directory.name
|
||||
result["score"] = int(risk.get("score") or 0)
|
||||
result["severity"] = (risk.get("severity") or "LOW").upper()
|
||||
result["recommendation"] = risk.get("recommendation") or "SAFE"
|
||||
result["issues"] = data.get("issues") or []
|
||||
result["report"] = data
|
||||
return result
|
||||
|
||||
|
||||
def _aggregate_sarif(results: list[dict]) -> dict:
|
||||
"""Merge per-skill SARIF-equivalent findings into one SARIF 2.1.0 log."""
|
||||
sarif_results: list[dict] = []
|
||||
severity_to_level = {
|
||||
"CRITICAL": "error",
|
||||
"HIGH": "error",
|
||||
"MEDIUM": "warning",
|
||||
"LOW": "note",
|
||||
}
|
||||
for res in results:
|
||||
if res.get("status") != "ok":
|
||||
continue
|
||||
skill_dir = res["path"].rstrip("/")
|
||||
for issue in res.get("issues", []):
|
||||
file_rel = issue.get("file") or ""
|
||||
# Prefix the issue's file (relative to the skill dir) with the
|
||||
# repo-relative skill dir so code scanning links resolve.
|
||||
if file_rel and not file_rel.startswith(skill_dir):
|
||||
uri = f"{skill_dir}/{file_rel.lstrip('./')}"
|
||||
else:
|
||||
uri = file_rel or skill_dir
|
||||
severity = (issue.get("severity") or "LOW").upper()
|
||||
region: dict = {}
|
||||
if issue.get("start_line"):
|
||||
region["startLine"] = issue["start_line"]
|
||||
if issue.get("end_line"):
|
||||
region["endLine"] = issue["end_line"]
|
||||
location = {"physicalLocation": {"artifactLocation": {"uri": uri}}}
|
||||
if region:
|
||||
location["physicalLocation"]["region"] = region
|
||||
sarif_results.append(
|
||||
{
|
||||
"ruleId": issue.get("rule_id") or "UNKNOWN",
|
||||
"level": severity_to_level.get(severity, "note"),
|
||||
"message": {"text": issue.get("message") or ""},
|
||||
"locations": [location],
|
||||
}
|
||||
)
|
||||
return {
|
||||
"$schema": "https://json.schemastore.org/sarif-2.1.0.json",
|
||||
"version": "2.1.0",
|
||||
"runs": [
|
||||
{
|
||||
"tool": {
|
||||
"driver": {
|
||||
"name": "skillspector",
|
||||
"informationUri": "https://github.com/NVIDIA/skillspector",
|
||||
"rules": [],
|
||||
}
|
||||
},
|
||||
"results": sarif_results,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _severity_emoji(severity: str) -> str:
|
||||
return {"LOW": "🟢", "MEDIUM": "🟡", "HIGH": "🔴", "CRITICAL": "🔴"}.get(severity, "")
|
||||
|
||||
|
||||
def build_markdown(results: list[dict], threshold: int) -> str:
|
||||
"""Produce the PR-comment Markdown summary."""
|
||||
scanned = [r for r in results if r.get("status") == "ok"]
|
||||
errors = [r for r in results if r.get("status") == "error"]
|
||||
flagged = sorted(
|
||||
[r for r in scanned if r["score"] > threshold],
|
||||
key=lambda r: r["score"],
|
||||
reverse=True,
|
||||
)
|
||||
high_critical = len(flagged)
|
||||
|
||||
sev_counts = {"LOW": 0, "MEDIUM": 0, "HIGH": 0, "CRITICAL": 0}
|
||||
for r in scanned:
|
||||
sev_counts[r["severity"]] = sev_counts.get(r["severity"], 0) + 1
|
||||
|
||||
status = "❌ ISSUES FOUND" if high_critical else "✅ PASSED"
|
||||
emoji = "⚠️" if high_critical else "🎉"
|
||||
|
||||
lines = [COMMENT_MARKER]
|
||||
lines.append(f"## {emoji} SkillSpector Security Scan")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"Static analysis (`--no-llm`) by "
|
||||
"[SkillSpector](https://github.com/NVIDIA/skillspector) (NVIDIA, Apache-2.0)."
|
||||
)
|
||||
lines.append("")
|
||||
lines.append(f"**Status:** {status}")
|
||||
lines.append("")
|
||||
lines.append("| Metric | Count |")
|
||||
lines.append("|--------|-------|")
|
||||
lines.append(f"| Skills scanned | {len(scanned)} |")
|
||||
lines.append(f"| 🔴 HIGH/CRITICAL (score > {threshold}) | {high_critical} |")
|
||||
lines.append(f"| 🟡 MEDIUM | {sev_counts['MEDIUM']} |")
|
||||
lines.append(f"| 🟢 LOW / clean | {sev_counts['LOW']} |")
|
||||
if errors:
|
||||
lines.append(f"| ⚠️ Scan errors | {len(errors)} |")
|
||||
lines.append("")
|
||||
|
||||
if flagged:
|
||||
lines.append("### 🔴 Flagged skills")
|
||||
lines.append("")
|
||||
lines.append("| Skill | Score | Severity | Recommendation | Issues | Top findings |")
|
||||
lines.append("|-------|-------|----------|----------------|--------|--------------|")
|
||||
for r in flagged[:30]:
|
||||
top_rules = ", ".join(
|
||||
sorted({str(i.get("rule_id") or "?") for i in r.get("issues", [])})[:5]
|
||||
)
|
||||
rec = str(r.get("recommendation", "")).replace("_", " ")
|
||||
lines.append(
|
||||
f"| `{r['name']}` | {r['score']}/100 | "
|
||||
f"{_severity_emoji(r['severity'])} {r['severity']} | {rec} | "
|
||||
f"{len(r.get('issues', []))} | {top_rules} |"
|
||||
)
|
||||
if len(flagged) > 30:
|
||||
lines.append("")
|
||||
lines.append(f"_...and {len(flagged) - 30} more flagged skill(s)._")
|
||||
lines.append("")
|
||||
|
||||
if errors:
|
||||
lines.append("### ⚠️ Scan errors")
|
||||
lines.append("")
|
||||
for r in errors[:10]:
|
||||
lines.append(f"- `{r['path']}` — {r.get('error', 'unknown error')}")
|
||||
if len(errors) > 10:
|
||||
lines.append(f"- _...and {len(errors) - 10} more._")
|
||||
lines.append("")
|
||||
|
||||
if not flagged and not errors:
|
||||
lines.append("No HIGH/CRITICAL findings. ✅")
|
||||
lines.append("")
|
||||
|
||||
lines.append("---")
|
||||
lines.append(
|
||||
"_Generated by SkillSpector. Score bands: 0-20 LOW, 21-50 MEDIUM, "
|
||||
"51-80 HIGH, 81-100 CRITICAL._"
|
||||
)
|
||||
|
||||
text = "\n".join(lines)
|
||||
if len(text) > MAX_COMMENT_CHARS:
|
||||
text = text[: MAX_COMMENT_CHARS - 200] + "\n\n_...report truncated (too long)._"
|
||||
return text
|
||||
|
||||
|
||||
def _write_output(name: str, value: str) -> None:
|
||||
"""Append a key=value pair to $GITHUB_OUTPUT (no-op if unset)."""
|
||||
out_path = os.environ.get("GITHUB_OUTPUT")
|
||||
if not out_path:
|
||||
return
|
||||
with open(out_path, "a", encoding="utf-8") as fh:
|
||||
if "\n" in value:
|
||||
fh.write(f"{name}<<__EOF__\n{value}\n__EOF__\n")
|
||||
else:
|
||||
fh.write(f"{name}={value}\n")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Batch SkillSpector scanner.")
|
||||
scope = parser.add_mutually_exclusive_group(required=True)
|
||||
scope.add_argument("--all", action="store_true", help="Scan every skill dir.")
|
||||
scope.add_argument(
|
||||
"--changed", action="store_true", help="Scan only skills changed vs --base-ref."
|
||||
)
|
||||
parser.add_argument("--base-ref", default="origin/main", help="Base ref for --changed.")
|
||||
parser.add_argument("--threshold", type=int, default=DEFAULT_THRESHOLD)
|
||||
parser.add_argument(
|
||||
"--fail-on-risk",
|
||||
action="store_true",
|
||||
help="Set output failed=true (and exit 1) if any skill exceeds threshold.",
|
||||
)
|
||||
parser.add_argument("--output-md", default="skillspector-report.md")
|
||||
parser.add_argument("--output-sarif", default="skillspector.sarif")
|
||||
parser.add_argument(
|
||||
"--dry-run", action="store_true", help="List discovered skills without scanning."
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.changed:
|
||||
skills = discover_changed_skills(args.base_ref)
|
||||
else:
|
||||
skills = discover_all_skills()
|
||||
|
||||
print(f"Discovered {len(skills)} skill director{'y' if len(skills) == 1 else 'ies'}.")
|
||||
|
||||
if args.dry_run:
|
||||
for s in skills:
|
||||
print(f" {s}")
|
||||
return 0
|
||||
|
||||
if not skills:
|
||||
# Nothing to scan is a pass.
|
||||
Path(args.output_md).write_text(
|
||||
f"{COMMENT_MARKER}\n## 🎉 SkillSpector Security Scan\n\n"
|
||||
"No skill components were changed in this PR.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
Path(args.output_sarif).write_text(json.dumps(_aggregate_sarif([]), indent=2))
|
||||
_write_output("scanned", "0")
|
||||
_write_output("high_critical_count", "0")
|
||||
_write_output("errors", "0")
|
||||
_write_output("failed", "false")
|
||||
return 0
|
||||
|
||||
results: list[dict] = []
|
||||
for idx, skill in enumerate(skills, 1):
|
||||
print(f"[{idx}/{len(skills)}] scanning {skill}")
|
||||
results.append(scan_skill(skill))
|
||||
|
||||
scanned = [r for r in results if r.get("status") == "ok"]
|
||||
errors = [r for r in results if r.get("status") == "error"]
|
||||
flagged = [r for r in scanned if r["score"] > args.threshold]
|
||||
|
||||
markdown = build_markdown(results, args.threshold)
|
||||
Path(args.output_md).write_text(markdown, encoding="utf-8")
|
||||
Path(args.output_sarif).write_text(
|
||||
json.dumps(_aggregate_sarif(results), indent=2), encoding="utf-8"
|
||||
)
|
||||
|
||||
print(
|
||||
f"Scanned={len(scanned)} flagged={len(flagged)} errors={len(errors)} "
|
||||
f"(threshold={args.threshold})"
|
||||
)
|
||||
|
||||
failed = bool(flagged) and args.fail_on_risk
|
||||
_write_output("scanned", str(len(scanned)))
|
||||
_write_output("high_critical_count", str(len(flagged)))
|
||||
_write_output("errors", str(len(errors)))
|
||||
_write_output("failed", "true" if failed else "false")
|
||||
|
||||
if failed:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
|
||||
# sync-api.sh - Sync API files from /api to /docs/api
|
||||
# This ensures serverless functions are deployed correctly with Vercel
|
||||
|
||||
set -e
|
||||
|
||||
echo "🔄 Syncing API files from /api to /docs/api..."
|
||||
echo ""
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Check if source files exist
|
||||
if [ ! -f "api/track-download-supabase.js" ]; then
|
||||
echo "❌ Error: api/track-download-supabase.js not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "api/package.json" ]; then
|
||||
echo "❌ Error: api/package.json not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create docs/api directory if it doesn't exist
|
||||
mkdir -p docs/api
|
||||
|
||||
# Copy files
|
||||
echo "${BLUE}📄 Copying track-download-supabase.js...${NC}"
|
||||
cp api/track-download-supabase.js docs/api/
|
||||
echo "${GREEN}✓ Copied track-download-supabase.js${NC}"
|
||||
|
||||
echo "${BLUE}📦 Copying package.json...${NC}"
|
||||
cp api/package.json docs/api/
|
||||
echo "${GREEN}✓ Copied package.json${NC}"
|
||||
|
||||
echo ""
|
||||
echo "${GREEN}✅ Sync completed successfully!${NC}"
|
||||
echo ""
|
||||
echo "📁 Files synced to docs/api/:"
|
||||
ls -lh docs/api/ | grep -E "(track-download-supabase.js|package.json)"
|
||||
|
||||
echo ""
|
||||
echo "📝 Next steps:"
|
||||
echo " 1. Review changes: git diff docs/api/"
|
||||
echo " 2. Commit changes: git add docs/api/ && git commit -m 'Sync API files'"
|
||||
echo " 3. Deploy to Vercel: vercel --prod"
|
||||
echo ""
|
||||
Reference in New Issue
Block a user