wehub-resource-sync
967 lines
50 KiB
Python
967 lines
50 KiB
Python
import os
|
|
import json
|
|
import shutil
|
|
import requests
|
|
import subprocess
|
|
from collections import defaultdict
|
|
from dotenv import load_dotenv
|
|
from pathlib import Path
|
|
|
|
# Load environment variables
|
|
load_dotenv()
|
|
|
|
def run_security_validation():
|
|
"""
|
|
Run security validation on all components and return the results.
|
|
Returns a dictionary with component paths as keys and security data as values.
|
|
"""
|
|
print("🔒 Running security validation on components...")
|
|
|
|
try:
|
|
# Change to cli-tool directory to run npm command
|
|
cli_tool_dir = Path(__file__).parent / 'cli-tool'
|
|
|
|
# Run security audit and generate JSON report
|
|
result = subprocess.run(
|
|
['npm', 'run', 'security-audit:json'],
|
|
cwd=cli_tool_dir,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=300 # 5 minute timeout
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
print(f"⚠️ Security validation completed with warnings")
|
|
print(f" stdout: {result.stdout[:200]}")
|
|
print(f" stderr: {result.stderr[:200]}")
|
|
else:
|
|
print("✅ Security validation completed successfully")
|
|
|
|
# Read the generated security report
|
|
security_report_path = cli_tool_dir / 'security-report.json'
|
|
|
|
if not security_report_path.exists():
|
|
print("⚠️ Security report not found, skipping security metadata")
|
|
return {}
|
|
|
|
with open(security_report_path, 'r', encoding='utf-8') as f:
|
|
security_data = json.load(f)
|
|
|
|
# Transform security data into a lookup dictionary
|
|
# Key format: "agents/category/name" -> security metadata
|
|
security_lookup = {}
|
|
|
|
for component_result in security_data.get('components', []):
|
|
component_info = component_result.get('component', {})
|
|
component_path = component_info.get('path', '')
|
|
component_type = component_info.get('type', '')
|
|
|
|
if component_path:
|
|
# Extract category and name from path
|
|
# Path format: components/agents/development-team/frontend-developer.md
|
|
path_parts = component_path.replace('\\', '/').split('/')
|
|
|
|
# Handle both formats: "components/agents/..." and "cli-tool/components/agents/..."
|
|
if 'components' in path_parts:
|
|
components_idx = path_parts.index('components')
|
|
if len(path_parts) > components_idx + 3:
|
|
component_type = path_parts[components_idx + 1] # agents, commands, etc.
|
|
category = path_parts[components_idx + 2]
|
|
file_name = path_parts[components_idx + 3]
|
|
name = os.path.splitext(file_name)[0]
|
|
|
|
# Create lookup key
|
|
key = f"{component_type}/{category}/{name}"
|
|
|
|
# Extract security metadata
|
|
overall = component_result.get('overall', {})
|
|
validators = component_result.get('validators', {})
|
|
|
|
# Build validators object with detailed errors and warnings
|
|
validators_data = {}
|
|
for validator_name, validator_result in validators.items():
|
|
# Process errors to extract line/column information
|
|
processed_errors = []
|
|
for error in validator_result.get('errors', []):
|
|
error_data = {
|
|
'level': error.get('level', 'error'),
|
|
'code': error.get('code', ''),
|
|
'message': error.get('message', ''),
|
|
'timestamp': error.get('timestamp', '')
|
|
}
|
|
|
|
# Extract metadata with line/column info
|
|
metadata = error.get('metadata', {})
|
|
if metadata:
|
|
error_data['metadata'] = metadata
|
|
|
|
# Extract location info if available
|
|
if 'line' in metadata:
|
|
error_data['line'] = metadata['line']
|
|
if 'column' in metadata:
|
|
error_data['column'] = metadata['column']
|
|
if 'position' in metadata:
|
|
error_data['position'] = metadata['position']
|
|
if 'lineText' in metadata:
|
|
error_data['lineText'] = metadata['lineText']
|
|
|
|
# Extract examples array if present (for patterns with multiple matches)
|
|
if 'examples' in metadata:
|
|
error_data['examples'] = metadata['examples']
|
|
|
|
processed_errors.append(error_data)
|
|
|
|
# Process warnings similarly
|
|
processed_warnings = []
|
|
for warning in validator_result.get('warnings', []):
|
|
warning_data = {
|
|
'level': warning.get('level', 'warning'),
|
|
'code': warning.get('code', ''),
|
|
'message': warning.get('message', ''),
|
|
'timestamp': warning.get('timestamp', '')
|
|
}
|
|
|
|
# Extract metadata with line/column info
|
|
metadata = warning.get('metadata', {})
|
|
if metadata:
|
|
warning_data['metadata'] = metadata
|
|
|
|
# Extract location info if available
|
|
if 'line' in metadata:
|
|
warning_data['line'] = metadata['line']
|
|
if 'column' in metadata:
|
|
warning_data['column'] = metadata['column']
|
|
if 'position' in metadata:
|
|
warning_data['position'] = metadata['position']
|
|
if 'lineText' in metadata:
|
|
warning_data['lineText'] = metadata['lineText']
|
|
|
|
# Extract examples array if present
|
|
if 'examples' in metadata:
|
|
warning_data['examples'] = metadata['examples']
|
|
|
|
processed_warnings.append(warning_data)
|
|
|
|
validators_data[validator_name] = {
|
|
'valid': validator_result.get('valid', False),
|
|
'score': validator_result.get('score', 0),
|
|
'errorCount': validator_result.get('errorCount', 0),
|
|
'warningCount': validator_result.get('warningCount', 0),
|
|
'errors': processed_errors,
|
|
'warnings': processed_warnings,
|
|
'info': validator_result.get('info', [])
|
|
}
|
|
|
|
security_lookup[key] = {
|
|
'validated': True,
|
|
'valid': overall.get('valid', False),
|
|
'score': overall.get('score', 0),
|
|
'errorCount': overall.get('errorCount', 0),
|
|
'warningCount': overall.get('warningCount', 0),
|
|
'lastValidated': security_data.get('timestamp', ''),
|
|
'validators': validators_data
|
|
}
|
|
|
|
# Add hash if available from integrity validator
|
|
integrity_data = validators.get('integrity', {})
|
|
if 'info' in integrity_data:
|
|
for info_item in integrity_data['info']:
|
|
# info_item is a dictionary with code, message, metadata
|
|
if isinstance(info_item, dict):
|
|
metadata = info_item.get('metadata', {})
|
|
if 'fullHash' in metadata:
|
|
security_lookup[key]['hash'] = metadata['fullHash']
|
|
break
|
|
elif 'hash' in metadata:
|
|
security_lookup[key]['hash'] = metadata['hash']
|
|
break
|
|
|
|
print(f"✅ Security metadata extracted for {len(security_lookup)} components")
|
|
return security_lookup
|
|
|
|
except subprocess.TimeoutExpired:
|
|
print("⚠️ Security validation timed out after 5 minutes")
|
|
return {}
|
|
except FileNotFoundError:
|
|
print("⚠️ npm command not found, skipping security validation")
|
|
return {}
|
|
except Exception as e:
|
|
print(f"⚠️ Error running security validation: {e}")
|
|
return {}
|
|
|
|
def fetch_download_stats():
|
|
"""
|
|
Fetch download statistics from Supabase
|
|
Returns a dictionary with component_type-component_name as key and download count as value
|
|
Supports: agents, commands, mcps, settings, hooks, sandbox, skills, templates, plugins
|
|
"""
|
|
print("📊 Fetching download statistics from Supabase...")
|
|
|
|
# Define type mapping once (DRY principle)
|
|
TYPE_MAPPING = {
|
|
'agent': 'agents',
|
|
'command': 'commands',
|
|
'setting': 'settings',
|
|
'hook': 'hooks',
|
|
'mcp': 'mcps',
|
|
'skill': 'skills',
|
|
'template': 'templates',
|
|
'plugin': 'plugins',
|
|
'sandbox': 'sandbox'
|
|
}
|
|
|
|
# Get Supabase credentials (same as generate_trending_data.py)
|
|
supabase_url = os.getenv("SUPABASE_URL")
|
|
supabase_api_key = os.getenv("SUPABASE_API_KEY")
|
|
|
|
if not supabase_url or not supabase_api_key:
|
|
print("⚠️ Warning: Missing Supabase credentials, skipping download stats")
|
|
return {}
|
|
|
|
try:
|
|
headers = {
|
|
'apikey': supabase_api_key,
|
|
'Authorization': f'Bearer {supabase_api_key}'
|
|
}
|
|
|
|
# First, let's query component_downloads to aggregate counts per component
|
|
# We need to handle pagination since there can be many records
|
|
api_url = f"{supabase_url}/rest/v1/component_downloads"
|
|
|
|
all_downloads = []
|
|
offset = 0
|
|
limit = 1000
|
|
|
|
# Fetch all records with pagination
|
|
max_pages = 1000 # Safety limit (1000 pages * 1000 records = 1,000,000 max)
|
|
for page in range(max_pages):
|
|
paginated_headers = headers.copy()
|
|
paginated_headers['Range'] = f'{offset}-{offset + limit - 1}'
|
|
|
|
response = requests.get(api_url, headers=paginated_headers)
|
|
|
|
if response.status_code not in [200, 206]:
|
|
print(f" Page {page+1}: Got status {response.status_code}, stopping")
|
|
break
|
|
|
|
batch = response.json()
|
|
if not batch:
|
|
break
|
|
|
|
all_downloads.extend(batch)
|
|
|
|
# Check if we have more records to fetch
|
|
content_range = response.headers.get('content-range', '')
|
|
|
|
# If we get a range like "45000-45977/*", check if we got less than limit records
|
|
if len(batch) < limit:
|
|
break # We've reached the end
|
|
|
|
# Also check for explicit total if provided
|
|
if content_range and '/' in content_range:
|
|
parts = content_range.split('/')
|
|
if parts[1] != '*':
|
|
try:
|
|
total = int(parts[1])
|
|
if offset + limit >= total:
|
|
break
|
|
except ValueError:
|
|
pass
|
|
|
|
offset += limit
|
|
|
|
# Progress indicator every 10 pages
|
|
if (page + 1) % 10 == 0:
|
|
print(f" Fetched {len(all_downloads)} records so far...")
|
|
|
|
print(f"📊 Total records fetched: {len(all_downloads)}")
|
|
|
|
# If we fetched records, use them
|
|
if len(all_downloads) > 0:
|
|
# Process component_downloads data
|
|
downloads = all_downloads
|
|
|
|
# Aggregate downloads by component
|
|
download_counts = {}
|
|
component_totals = defaultdict(int)
|
|
|
|
for download in downloads:
|
|
component_type = download.get('component_type', '')
|
|
component_name = download.get('component_name', '')
|
|
|
|
if component_type and component_name:
|
|
# Handle case where component_name already includes category
|
|
if '/' in component_name:
|
|
category = component_name.split('/')[0]
|
|
actual_name = component_name.split('/')[-1]
|
|
else:
|
|
actual_name = component_name
|
|
category = 'general'
|
|
|
|
# Create a key for aggregation matching trending data structure
|
|
key = f"{component_type}|{category}|{actual_name}"
|
|
component_totals[key] += 1
|
|
|
|
# Convert to the format we need using the TYPE_MAPPING constant
|
|
for key, count in component_totals.items():
|
|
parts = key.split('|')
|
|
if len(parts) == 3:
|
|
component_type, category, component_name = parts
|
|
mapped_type = TYPE_MAPPING.get(component_type, component_type + 's')
|
|
final_key = f"{mapped_type}/{category}/{component_name}"
|
|
download_counts[final_key] = count
|
|
|
|
print(f"✅ Fetched and aggregated {len(download_counts)} component download stats")
|
|
return download_counts
|
|
else:
|
|
# Try alternative: fetch from download_stats table if it exists
|
|
print("⚠️ No data from component_downloads, trying download_stats table...")
|
|
alt_url = f"{supabase_url}/rest/v1/download_stats"
|
|
alt_response = requests.get(alt_url, headers=headers)
|
|
|
|
if alt_response.status_code == 200:
|
|
print("📊 Using download_stats table instead...")
|
|
stats = alt_response.json()
|
|
download_counts = {}
|
|
|
|
for stat in stats:
|
|
component_type = stat.get('component_type', '')
|
|
component_name = stat.get('component_name', '')
|
|
total_downloads = stat.get('total_downloads', 0)
|
|
|
|
# Handle case where component_name already includes category
|
|
if '/' in component_name:
|
|
category = component_name.split('/')[0]
|
|
actual_name = component_name.split('/')[-1]
|
|
else:
|
|
actual_name = component_name
|
|
category = 'general'
|
|
|
|
# Map to plural form using TYPE_MAPPING constant
|
|
mapped_type = TYPE_MAPPING.get(component_type, component_type + 's')
|
|
key = f"{mapped_type}/{category}/{actual_name}"
|
|
download_counts[key] = total_downloads
|
|
|
|
print(f"✅ Fetched stats for {len(download_counts)} components from download_stats")
|
|
return download_counts
|
|
else:
|
|
print("⚠️ No download stats available")
|
|
return {}
|
|
|
|
except Exception as e:
|
|
print(f"⚠️ Error fetching download stats: {e}")
|
|
return {}
|
|
|
|
def scan_directory_recursively(directory_path, relative_to_path=None):
|
|
"""
|
|
Recursively scan a directory and return all files with their relative paths.
|
|
"""
|
|
files_list = []
|
|
|
|
if not os.path.isdir(directory_path):
|
|
return files_list
|
|
|
|
for root, dirs, files in os.walk(directory_path):
|
|
for file in files:
|
|
file_path = os.path.join(root, file)
|
|
if relative_to_path:
|
|
relative_path = os.path.relpath(file_path, relative_to_path)
|
|
files_list.append(relative_path.replace("\\", "/"))
|
|
else:
|
|
files_list.append(file)
|
|
|
|
return files_list
|
|
|
|
def generate_components_json():
|
|
"""
|
|
Scans the cli-tool/components and cli-tool/templates directories and generates a components.json file
|
|
for the static website, including the content of each file and download statistics.
|
|
"""
|
|
components_base_path = 'cli-tool/components'
|
|
templates_base_path = 'cli-tool/templates'
|
|
plugins_path = '.claude-plugin/marketplace.json'
|
|
output_path = 'docs/components.json'
|
|
components_data = {'agents': [], 'commands': [], 'mcps': [], 'settings': [], 'hooks': [], 'sandbox': [], 'skills': [], 'loops': [], 'templates': [], 'plugins': []}
|
|
|
|
# Run security validation
|
|
security_metadata = run_security_validation()
|
|
|
|
# Fetch download statistics
|
|
download_stats = fetch_download_stats()
|
|
component_types = ['agents', 'commands', 'mcps', 'settings', 'hooks', 'sandbox', 'skills', 'loops']
|
|
|
|
print(f"Starting scan of {components_base_path} and {templates_base_path}...")
|
|
|
|
# Scan components (existing logic)
|
|
for component_type in component_types:
|
|
type_path = os.path.join(components_base_path, component_type)
|
|
if not os.path.isdir(type_path):
|
|
print(f"Warning: Directory not found for type: {component_type}")
|
|
continue
|
|
|
|
print(f"Scanning for {component_type} in {type_path}...")
|
|
|
|
# Special handling for skills - they use SKILL.md inside category/skill directories
|
|
if component_type == 'skills':
|
|
for category in os.listdir(type_path):
|
|
category_path = os.path.join(type_path, category)
|
|
if os.path.isdir(category_path) and not category.endswith('.md'):
|
|
for skill_dir in os.listdir(category_path):
|
|
skill_dir_path = os.path.join(category_path, skill_dir)
|
|
if os.path.isdir(skill_dir_path):
|
|
# Look for SKILL.md inside the skill directory
|
|
skill_file_path = os.path.join(skill_dir_path, 'SKILL.md')
|
|
if os.path.isfile(skill_file_path):
|
|
name = skill_dir # Use directory name as skill name
|
|
|
|
# Read file content
|
|
content = ''
|
|
description = ''
|
|
try:
|
|
with open(skill_file_path, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
|
|
# Extract metadata from frontmatter if available
|
|
author = ''
|
|
repo = ''
|
|
version = ''
|
|
license_field = ''
|
|
keywords = []
|
|
if content.startswith('---'):
|
|
frontmatter_end = content.find('---', 3)
|
|
if frontmatter_end != -1:
|
|
frontmatter = content[3:frontmatter_end]
|
|
for line in frontmatter.split('\n'):
|
|
if line.startswith('description:'):
|
|
description = line.split('description:', 1)[1].strip()
|
|
elif line.startswith('author:'):
|
|
author = line.split('author:', 1)[1].strip()
|
|
elif line.startswith('repo:'):
|
|
repo = line.split('repo:', 1)[1].strip()
|
|
elif line.startswith('version:'):
|
|
version = line.split('version:', 1)[1].strip()
|
|
elif line.startswith('license:'):
|
|
license_field = line.split('license:', 1)[1].strip()
|
|
elif line.startswith('tags:'):
|
|
tags_str = line.split('tags:', 1)[1].strip()
|
|
# Parse tags array [tag1, tag2, tag3]
|
|
if tags_str.startswith('[') and tags_str.endswith(']'):
|
|
tags_str = tags_str[1:-1]
|
|
keywords = [tag.strip() for tag in tags_str.split(',')]
|
|
|
|
except Exception as e:
|
|
print(f"Warning: Could not read file {skill_file_path}: {e}")
|
|
|
|
# Look up download count for this skill
|
|
download_key = f"{component_type}/{category}/{name}"
|
|
downloads = download_stats.get(download_key, 0)
|
|
|
|
# Look up security metadata for this skill
|
|
security_key = f"{component_type}/{category}/{name}"
|
|
security = security_metadata.get(security_key, {
|
|
'validated': False,
|
|
'valid': None,
|
|
'score': None,
|
|
'errorCount': 0,
|
|
'warningCount': 0,
|
|
'lastValidated': None
|
|
})
|
|
|
|
# Collect reference files (all files except SKILL.md)
|
|
references = []
|
|
for ref_file in scan_directory_recursively(skill_dir_path, skill_dir_path):
|
|
if ref_file != 'SKILL.md':
|
|
references.append(ref_file)
|
|
references.sort()
|
|
|
|
# Path includes category for proper organization
|
|
component = {
|
|
'name': name, # Just the skill directory name
|
|
'path': f"{category}/{name}", # category/skill-name format
|
|
'category': category,
|
|
'type': 'skill',
|
|
'content': content,
|
|
'description': description,
|
|
'author': author,
|
|
'repo': repo,
|
|
'version': version,
|
|
'license': license_field,
|
|
'keywords': keywords,
|
|
'downloads': downloads,
|
|
'security': security,
|
|
'references': references
|
|
}
|
|
components_data[component_type].append(component)
|
|
print(f" Processed skill: {category}/{name}")
|
|
continue # Skip the normal file scanning for skills
|
|
|
|
# Normal scanning for other component types
|
|
for category in os.listdir(type_path):
|
|
category_path = os.path.join(type_path, category)
|
|
if os.path.isdir(category_path):
|
|
for file_name in os.listdir(category_path):
|
|
file_path = os.path.join(category_path, file_name)
|
|
if os.path.isfile(file_path) and (file_name.endswith('.md') or file_name.endswith('.json')) and not file_name.endswith('.py'):
|
|
name = os.path.splitext(file_name)[0]
|
|
|
|
# Read file content
|
|
content = ''
|
|
description = ''
|
|
author = ''
|
|
repo = ''
|
|
version = ''
|
|
license_field = ''
|
|
keywords = []
|
|
try:
|
|
with open(file_path, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
|
|
# Extract description, author and repo from JSON files
|
|
if file_name.endswith('.json'):
|
|
try:
|
|
import json
|
|
json_data = json.loads(content)
|
|
|
|
if component_type == 'mcps':
|
|
# Extract description from the first mcpServer entry
|
|
if 'mcpServers' in json_data:
|
|
for server_name, server_config in json_data['mcpServers'].items():
|
|
if isinstance(server_config, dict) and 'description' in server_config:
|
|
description = server_config['description']
|
|
break # Use the first description found
|
|
elif component_type in ['settings', 'hooks']:
|
|
# Extract metadata from settings/hooks JSON files
|
|
if 'description' in json_data:
|
|
description = json_data['description']
|
|
if 'author' in json_data:
|
|
author = json_data['author']
|
|
if 'repo' in json_data:
|
|
repo = json_data['repo']
|
|
if 'version' in json_data:
|
|
version = json_data['version']
|
|
if 'license' in json_data:
|
|
license_field = json_data['license']
|
|
if 'keywords' in json_data and isinstance(json_data['keywords'], list):
|
|
keywords = json_data['keywords']
|
|
|
|
except json.JSONDecodeError:
|
|
print(f"Warning: Invalid JSON in {file_path}")
|
|
|
|
# Extract metadata from markdown frontmatter
|
|
elif file_name.endswith('.md'):
|
|
if content.startswith('---'):
|
|
frontmatter_end = content.find('---', 3)
|
|
if frontmatter_end != -1:
|
|
frontmatter = content[3:frontmatter_end]
|
|
for line in frontmatter.split('\n'):
|
|
if line.startswith('description:'):
|
|
description = line.split('description:', 1)[1].strip()
|
|
elif line.startswith('author:'):
|
|
author = line.split('author:', 1)[1].strip()
|
|
elif line.startswith('repo:'):
|
|
repo = line.split('repo:', 1)[1].strip()
|
|
elif line.startswith('version:'):
|
|
version = line.split('version:', 1)[1].strip()
|
|
elif line.startswith('license:'):
|
|
license_field = line.split('license:', 1)[1].strip()
|
|
elif line.startswith('tags:'):
|
|
tags_str = line.split('tags:', 1)[1].strip()
|
|
# Parse tags array [tag1, tag2, tag3]
|
|
if tags_str.startswith('[') and tags_str.endswith(']'):
|
|
tags_str = tags_str[1:-1]
|
|
keywords = [tag.strip() for tag in tags_str.split(',')]
|
|
|
|
except Exception as e:
|
|
print(f"Warning: Could not read file {file_path}: {e}")
|
|
|
|
# Look up download count for this component
|
|
# The key in download_stats uses the plural form: agents/category/name
|
|
download_key = f"{component_type}/{category}/{name}"
|
|
downloads = download_stats.get(download_key, 0)
|
|
|
|
# Look up security metadata for this component
|
|
security_key = f"{component_type}/{category}/{name}"
|
|
security = security_metadata.get(security_key, {
|
|
'validated': False,
|
|
'valid': None,
|
|
'score': None,
|
|
'errorCount': 0,
|
|
'warningCount': 0,
|
|
'lastValidated': None
|
|
})
|
|
|
|
component = {
|
|
'name': name,
|
|
'path': os.path.join(category, file_name).replace("\\", "/"),
|
|
'category': category,
|
|
'type': component_type[:-1], # singular form
|
|
'content': content, # Add file content
|
|
'description': description, # Add description for MCPs
|
|
'author': author, # Add author field
|
|
'repo': repo, # Add repository field
|
|
'version': version, # Add version field
|
|
'license': license_field, # Add license field
|
|
'keywords': keywords, # Add keywords field
|
|
'downloads': downloads, # Add download count
|
|
'security': security # Add security metadata
|
|
}
|
|
components_data[component_type].append(component)
|
|
|
|
# Scan templates (new logic)
|
|
if os.path.isdir(templates_base_path):
|
|
print(f"Scanning for templates in {templates_base_path}...")
|
|
|
|
for language_dir in os.listdir(templates_base_path):
|
|
language_path = os.path.join(templates_base_path, language_dir)
|
|
if os.path.isdir(language_path):
|
|
print(f" Processing language: {language_dir}")
|
|
|
|
# Collect files in the language directory (excluding examples folder)
|
|
language_files = []
|
|
for item in os.listdir(language_path):
|
|
item_path = os.path.join(language_path, item)
|
|
if os.path.isfile(item_path):
|
|
language_files.append(item)
|
|
elif os.path.isdir(item_path) and item != 'examples':
|
|
# For directories like .claude, scan recursively
|
|
if item == '.claude':
|
|
recursive_files = scan_directory_recursively(item_path, language_path)
|
|
language_files.extend(recursive_files)
|
|
else:
|
|
# For other directories, just add the directory name (optional)
|
|
pass
|
|
|
|
# Look up download count for this template
|
|
template_download_key = f"templates/{language_dir}"
|
|
template_downloads = download_stats.get(template_download_key, 0)
|
|
|
|
# Create language template entry
|
|
language_template = {
|
|
'name': language_dir,
|
|
'id': language_dir,
|
|
'type': 'template',
|
|
'subtype': 'language',
|
|
'category': 'languages',
|
|
'description': f'{language_dir.title()} project template',
|
|
'files': language_files,
|
|
'installCommand': f'npx claude-code-templates@latest --template={language_dir} --yes',
|
|
'downloads': template_downloads
|
|
}
|
|
components_data['templates'].append(language_template)
|
|
|
|
# Check for examples folder (contains frameworks)
|
|
examples_path = os.path.join(language_path, 'examples')
|
|
if os.path.isdir(examples_path):
|
|
for framework_dir in os.listdir(examples_path):
|
|
framework_path = os.path.join(examples_path, framework_dir)
|
|
if os.path.isdir(framework_path):
|
|
print(f" Processing framework: {framework_dir}")
|
|
|
|
# Collect files in the framework directory
|
|
framework_files = []
|
|
for item in os.listdir(framework_path):
|
|
item_path = os.path.join(framework_path, item)
|
|
if os.path.isfile(item_path):
|
|
framework_files.append(item)
|
|
elif os.path.isdir(item_path):
|
|
# For directories like .claude, scan recursively
|
|
if item == '.claude':
|
|
recursive_files = scan_directory_recursively(item_path, framework_path)
|
|
framework_files.extend(recursive_files)
|
|
else:
|
|
# For other directories, just add the directory name (optional)
|
|
pass
|
|
|
|
# Look up download count for this framework
|
|
framework_download_key = f"templates/{framework_dir}"
|
|
framework_downloads = download_stats.get(framework_download_key, 0)
|
|
|
|
# Create framework template entry
|
|
framework_template = {
|
|
'name': framework_dir,
|
|
'id': framework_dir,
|
|
'type': 'template',
|
|
'subtype': 'framework',
|
|
'category': 'frameworks',
|
|
'language': language_dir,
|
|
'description': f'{framework_dir.title()} with {language_dir.title()}',
|
|
'files': framework_files,
|
|
'installCommand': f'npx claude-code-templates@latest --template={framework_dir} --yes',
|
|
'downloads': framework_downloads
|
|
}
|
|
components_data['templates'].append(framework_template)
|
|
else:
|
|
print(f"Warning: Templates directory not found: {templates_base_path}")
|
|
|
|
# Load components metadata from marketplace.json (Claude Code standard)
|
|
components_marketplace_path = 'cli-tool/components/.claude-plugin/marketplace.json'
|
|
components_marketplace = None
|
|
if os.path.isfile(components_marketplace_path):
|
|
print(f"Loading components metadata from {components_marketplace_path}...")
|
|
try:
|
|
with open(components_marketplace_path, 'r', encoding='utf-8') as f:
|
|
components_marketplace = json.load(f)
|
|
print(f"✅ Loaded metadata for {len(components_marketplace.get('agents', []))} agents")
|
|
except Exception as e:
|
|
print(f"⚠️ Error reading components metadata from {components_marketplace_path}: {e}")
|
|
|
|
# Scan plugins from marketplace.json and store marketplace data
|
|
marketplace_full_data = None
|
|
if os.path.isfile(plugins_path):
|
|
print(f"Scanning for plugins in {plugins_path}...")
|
|
try:
|
|
with open(plugins_path, 'r', encoding='utf-8') as f:
|
|
marketplace_data = json.load(f)
|
|
|
|
# Store full marketplace data for inclusion in components.json
|
|
marketplace_full_data = marketplace_data
|
|
|
|
if 'plugins' in marketplace_data:
|
|
for plugin in marketplace_data['plugins']:
|
|
plugin_name = plugin.get('name', '')
|
|
plugin_description = plugin.get('description', '')
|
|
plugin_version = plugin.get('version', '1.0.0')
|
|
plugin_keywords = plugin.get('keywords', [])
|
|
plugin_author = plugin.get('author', {}).get('name', '')
|
|
|
|
# Get component lists with file names
|
|
commands_list = plugin.get('commands', [])
|
|
agents_list = plugin.get('agents', [])
|
|
mcps_list = plugin.get('mcpServers', [])
|
|
|
|
# Extract component names with category/folder from file paths
|
|
def extract_component_with_category(path):
|
|
# Extract category/name from path like "./cli-tool/components/commands/git/feature.md"
|
|
# Result should be "git/feature"
|
|
parts = path.split('/')
|
|
if len(parts) >= 2:
|
|
filename = parts[-1].replace('.md', '').replace('.json', '')
|
|
category = parts[-2]
|
|
return f"{category}/{filename}"
|
|
else:
|
|
# Fallback to just filename if path structure is unexpected
|
|
filename = path.split('/')[-1]
|
|
return filename.replace('.md', '').replace('.json', '')
|
|
|
|
commands_names = [extract_component_with_category(cmd) for cmd in commands_list]
|
|
agents_names = [extract_component_with_category(agent) for agent in agents_list]
|
|
mcps_names = [extract_component_with_category(mcp) for mcp in mcps_list]
|
|
|
|
# Look up download count for this plugin
|
|
plugin_download_key = f"plugins/{plugin_name}"
|
|
plugin_downloads = download_stats.get(plugin_download_key, 0)
|
|
|
|
# Create plugin entry
|
|
plugin_entry = {
|
|
'name': plugin_name,
|
|
'id': plugin_name,
|
|
'type': 'plugin',
|
|
'description': plugin_description,
|
|
'version': plugin_version,
|
|
'keywords': plugin_keywords,
|
|
'author': plugin_author,
|
|
'commands': len(commands_list),
|
|
'agents': len(agents_list),
|
|
'mcpServers': len(mcps_list),
|
|
'commandsList': commands_names,
|
|
'agentsList': agents_names,
|
|
'mcpServersList': mcps_names,
|
|
'installCommand': f'/plugin install {plugin_name}@claude-code-templates',
|
|
'downloads': plugin_downloads
|
|
}
|
|
components_data['plugins'].append(plugin_entry)
|
|
print(f" Processed plugin: {plugin_name}")
|
|
|
|
print(f"✅ Processed {len(components_data['plugins'])} plugins from marketplace.json")
|
|
except Exception as e:
|
|
print(f"⚠️ Error reading plugins from {plugins_path}: {e}")
|
|
else:
|
|
print(f"Warning: Plugins file not found: {plugins_path}")
|
|
|
|
# Sort components alphabetically
|
|
for component_type in components_data:
|
|
if component_type in ['templates', 'plugins']:
|
|
# Sort templates and plugins by name since they don't have path
|
|
components_data[component_type].sort(key=lambda x: x['name'])
|
|
else:
|
|
# Sort other components by path
|
|
components_data[component_type].sort(key=lambda x: x['path'])
|
|
|
|
# Add marketplace metadata (root level - our public marketplace)
|
|
if marketplace_full_data:
|
|
components_data['marketplace'] = marketplace_full_data
|
|
print("✅ Added public marketplace metadata to components.json")
|
|
|
|
# Add components marketplace metadata (Claude Code standard)
|
|
if components_marketplace:
|
|
components_data['componentsMarketplace'] = components_marketplace
|
|
print("✅ Added components marketplace metadata to components.json")
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Performance optimization: separate content from the lightweight index.
|
|
#
|
|
# docs/components.json → index without 'content' (~87% smaller)
|
|
# dashboard/public/components.json → same, served by Cloudflare Pages
|
|
# dashboard/public/component-content/{type}/{slug}.json → per-component
|
|
# raw content, loaded on demand only on detail pages.
|
|
# ---------------------------------------------------------------------------
|
|
dashboard_public_dir = 'dashboard/public'
|
|
dashboard_content_dir = os.path.join(dashboard_public_dir, 'component-content')
|
|
|
|
# Component types that carry a 'content' field
|
|
content_bearing_types = ['agents', 'commands', 'mcps', 'settings', 'hooks', 'sandbox', 'skills', 'loops']
|
|
|
|
# 1. Write per-component content files directly to dashboard/public/component-content/
|
|
os.makedirs(dashboard_content_dir, exist_ok=True)
|
|
written_content_count = 0
|
|
for ctype in content_bearing_types:
|
|
for component in components_data.get(ctype, []):
|
|
raw_content = component.get('content', '')
|
|
if not raw_content:
|
|
continue
|
|
# Slug: path with extension stripped (matches detail-page URL lookup)
|
|
slug = component['path'].replace('.md', '').replace('.json', '')
|
|
content_file_path = os.path.join(dashboard_content_dir, ctype, slug + '.json')
|
|
os.makedirs(os.path.dirname(content_file_path), exist_ok=True)
|
|
try:
|
|
with open(content_file_path, 'w', encoding='utf-8') as f:
|
|
json.dump({'content': raw_content}, f, ensure_ascii=False)
|
|
written_content_count += 1
|
|
except IOError as e:
|
|
print(f"Warning: Could not write content file {content_file_path}: {e}")
|
|
|
|
print(f"Generated {written_content_count} component content files in {dashboard_content_dir}/")
|
|
|
|
# 2. Build the legacy/docs index: strip only 'content'.
|
|
# The legacy static site (docs/js/*) renders security badges, so the
|
|
# 'security' field MUST stay in docs/components.json.
|
|
index_data = {}
|
|
for k, v in components_data.items():
|
|
if k in content_bearing_types:
|
|
index_data[k] = [{key: val for key, val in c.items() if key != 'content'} for c in v]
|
|
else:
|
|
index_data[k] = v
|
|
|
|
# 3. Write the docs index (with 'security', without 'content').
|
|
try:
|
|
with open(output_path, 'w', encoding='utf-8') as f:
|
|
json.dump(index_data, f, indent=2, ensure_ascii=False)
|
|
print(f"Successfully generated {output_path} (index without content).")
|
|
except IOError as e:
|
|
print(f"Error writing to {output_path}: {e}")
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dashboard-specific artifacts (Cloudflare Pages, www.aitmpl.com).
|
|
#
|
|
# The dashboard never reads the 'security' field, so we strip it here to
|
|
# shrink the payload the browser downloads. We also emit smaller,
|
|
# purpose-built artifacts so islands stop downloading the whole index:
|
|
#
|
|
# components.json → full index, no 'content', no 'security'
|
|
# counts.json → { type: count } (sidebars/plugins pages)
|
|
# components/{type}.json → per-type slice (grid loads only active type)
|
|
# search-index.json → flat [{type,name,path,description,category}]
|
|
# ---------------------------------------------------------------------------
|
|
STRIP_KEYS = {'content', 'security'}
|
|
|
|
def strip_for_dashboard(component):
|
|
return {key: val for key, val in component.items() if key not in STRIP_KEYS}
|
|
|
|
# Build the dashboard index (content + security stripped from components).
|
|
dashboard_index_data = {}
|
|
for k, v in index_data.items():
|
|
if k in content_bearing_types and isinstance(v, list):
|
|
dashboard_index_data[k] = [strip_for_dashboard(c) for c in v]
|
|
elif isinstance(v, list):
|
|
# Non-content types (e.g. mcps/settings/hooks may carry security)
|
|
dashboard_index_data[k] = [strip_for_dashboard(c) for c in v]
|
|
else:
|
|
dashboard_index_data[k] = v
|
|
|
|
try:
|
|
os.makedirs(dashboard_public_dir, exist_ok=True)
|
|
dashboard_index_path = os.path.join(dashboard_public_dir, 'components.json')
|
|
with open(dashboard_index_path, 'w', encoding='utf-8') as f:
|
|
json.dump(dashboard_index_data, f, ensure_ascii=False)
|
|
print(f"Generated {dashboard_index_path} (dashboard index, no content/security).")
|
|
except Exception as e:
|
|
print(f"Warning: Could not write dashboard index: {e}")
|
|
|
|
# counts.json — tiny per-type counts so sidebars/plugins pages stop
|
|
# downloading the full index just to read array lengths.
|
|
try:
|
|
counts = {k: len(v) for k, v in dashboard_index_data.items() if isinstance(v, list)}
|
|
counts_path = os.path.join(dashboard_public_dir, 'counts.json')
|
|
with open(counts_path, 'w', encoding='utf-8') as f:
|
|
json.dump(counts, f, ensure_ascii=False)
|
|
print(f"Generated {counts_path} ({counts}).")
|
|
except Exception as e:
|
|
print(f"Warning: Could not write counts.json: {e}")
|
|
|
|
# components/{type}.json — per-type slices so the grid loads only the
|
|
# active type instead of all 1800+ components at once.
|
|
try:
|
|
per_type_dir = os.path.join(dashboard_public_dir, 'components')
|
|
os.makedirs(per_type_dir, exist_ok=True)
|
|
for k, v in dashboard_index_data.items():
|
|
if not isinstance(v, list):
|
|
continue
|
|
type_path = os.path.join(per_type_dir, f'{k}.json')
|
|
with open(type_path, 'w', encoding='utf-8') as f:
|
|
json.dump(v, f, ensure_ascii=False)
|
|
print(f"Generated per-type index files in {per_type_dir}/")
|
|
except Exception as e:
|
|
print(f"Warning: Could not write per-type index files: {e}")
|
|
|
|
# search-index.json — flat array with only the fields the global (Cmd+K)
|
|
# search reads: type, name, path, description, category.
|
|
try:
|
|
search_index = []
|
|
for k, v in dashboard_index_data.items():
|
|
if not isinstance(v, list):
|
|
continue
|
|
for c in v:
|
|
search_index.append({
|
|
'type': c.get('type'),
|
|
'name': c.get('name'),
|
|
'path': c.get('path'),
|
|
'description': c.get('description', ''),
|
|
'category': c.get('category'),
|
|
})
|
|
search_path = os.path.join(dashboard_public_dir, 'search-index.json')
|
|
with open(search_path, 'w', encoding='utf-8') as f:
|
|
json.dump(search_index, f, ensure_ascii=False)
|
|
print(f"Generated {search_path} ({len(search_index)} entries).")
|
|
except Exception as e:
|
|
print(f"Warning: Could not write search-index.json: {e}")
|
|
|
|
# Log summary (uses original components_data which still has content for accurate counts)
|
|
print("\n--- Generation Summary ---")
|
|
for component_type, components in components_data.items():
|
|
# Skip marketplace metadata in summary (it's not a component type)
|
|
if component_type in ['marketplace', 'componentsMarketplace']:
|
|
continue
|
|
|
|
print(f" - Found and processed {len(components)} {component_type}")
|
|
if component_type == 'templates':
|
|
languages = len([t for t in components if t.get('subtype') == 'language'])
|
|
frameworks = len([t for t in components if t.get('subtype') == 'framework'])
|
|
print(f" • {languages} languages, {frameworks} frameworks")
|
|
elif component_type == 'plugins':
|
|
total_commands = sum(p.get('commands', 0) for p in components)
|
|
total_agents = sum(p.get('agents', 0) for p in components)
|
|
total_mcps = sum(p.get('mcpServers', 0) for p in components)
|
|
print(f" • {total_commands} commands, {total_agents} agents, {total_mcps} MCPs")
|
|
|
|
# Security statistics for components with security metadata
|
|
if component_type not in ['templates', 'plugins']:
|
|
validated = len([c for c in components if c.get('security', {}).get('validated', False)])
|
|
valid_components = len([c for c in components if c.get('security', {}).get('valid', False)])
|
|
avg_score = sum(c.get('security', {}).get('score', 0) for c in components if c.get('security', {}).get('validated', False)) / validated if validated > 0 else 0
|
|
print(f" • Security: {validated} validated, {valid_components} passed, avg score: {avg_score:.1f}")
|
|
|
|
print("--------------------------")
|
|
|
|
if __name__ == '__main__':
|
|
generate_components_json()
|