chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,966 @@
|
||||
import os
|
||||
import json
|
||||
import shutil
|
||||
import requests
|
||||
import subprocess
|
||||
from collections import defaultdict
|
||||
from dotenv import load_dotenv
|
||||
from pathlib import Path
|
||||
|
||||
# Load environment variables
|
||||
load_dotenv()
|
||||
|
||||
def run_security_validation():
|
||||
"""
|
||||
Run security validation on all components and return the results.
|
||||
Returns a dictionary with component paths as keys and security data as values.
|
||||
"""
|
||||
print("🔒 Running security validation on components...")
|
||||
|
||||
try:
|
||||
# Change to cli-tool directory to run npm command
|
||||
cli_tool_dir = Path(__file__).parent / 'cli-tool'
|
||||
|
||||
# Run security audit and generate JSON report
|
||||
result = subprocess.run(
|
||||
['npm', 'run', 'security-audit:json'],
|
||||
cwd=cli_tool_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300 # 5 minute timeout
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"⚠️ Security validation completed with warnings")
|
||||
print(f" stdout: {result.stdout[:200]}")
|
||||
print(f" stderr: {result.stderr[:200]}")
|
||||
else:
|
||||
print("✅ Security validation completed successfully")
|
||||
|
||||
# Read the generated security report
|
||||
security_report_path = cli_tool_dir / 'security-report.json'
|
||||
|
||||
if not security_report_path.exists():
|
||||
print("⚠️ Security report not found, skipping security metadata")
|
||||
return {}
|
||||
|
||||
with open(security_report_path, 'r', encoding='utf-8') as f:
|
||||
security_data = json.load(f)
|
||||
|
||||
# Transform security data into a lookup dictionary
|
||||
# Key format: "agents/category/name" -> security metadata
|
||||
security_lookup = {}
|
||||
|
||||
for component_result in security_data.get('components', []):
|
||||
component_info = component_result.get('component', {})
|
||||
component_path = component_info.get('path', '')
|
||||
component_type = component_info.get('type', '')
|
||||
|
||||
if component_path:
|
||||
# Extract category and name from path
|
||||
# Path format: components/agents/development-team/frontend-developer.md
|
||||
path_parts = component_path.replace('\\', '/').split('/')
|
||||
|
||||
# Handle both formats: "components/agents/..." and "cli-tool/components/agents/..."
|
||||
if 'components' in path_parts:
|
||||
components_idx = path_parts.index('components')
|
||||
if len(path_parts) > components_idx + 3:
|
||||
component_type = path_parts[components_idx + 1] # agents, commands, etc.
|
||||
category = path_parts[components_idx + 2]
|
||||
file_name = path_parts[components_idx + 3]
|
||||
name = os.path.splitext(file_name)[0]
|
||||
|
||||
# Create lookup key
|
||||
key = f"{component_type}/{category}/{name}"
|
||||
|
||||
# Extract security metadata
|
||||
overall = component_result.get('overall', {})
|
||||
validators = component_result.get('validators', {})
|
||||
|
||||
# Build validators object with detailed errors and warnings
|
||||
validators_data = {}
|
||||
for validator_name, validator_result in validators.items():
|
||||
# Process errors to extract line/column information
|
||||
processed_errors = []
|
||||
for error in validator_result.get('errors', []):
|
||||
error_data = {
|
||||
'level': error.get('level', 'error'),
|
||||
'code': error.get('code', ''),
|
||||
'message': error.get('message', ''),
|
||||
'timestamp': error.get('timestamp', '')
|
||||
}
|
||||
|
||||
# Extract metadata with line/column info
|
||||
metadata = error.get('metadata', {})
|
||||
if metadata:
|
||||
error_data['metadata'] = metadata
|
||||
|
||||
# Extract location info if available
|
||||
if 'line' in metadata:
|
||||
error_data['line'] = metadata['line']
|
||||
if 'column' in metadata:
|
||||
error_data['column'] = metadata['column']
|
||||
if 'position' in metadata:
|
||||
error_data['position'] = metadata['position']
|
||||
if 'lineText' in metadata:
|
||||
error_data['lineText'] = metadata['lineText']
|
||||
|
||||
# Extract examples array if present (for patterns with multiple matches)
|
||||
if 'examples' in metadata:
|
||||
error_data['examples'] = metadata['examples']
|
||||
|
||||
processed_errors.append(error_data)
|
||||
|
||||
# Process warnings similarly
|
||||
processed_warnings = []
|
||||
for warning in validator_result.get('warnings', []):
|
||||
warning_data = {
|
||||
'level': warning.get('level', 'warning'),
|
||||
'code': warning.get('code', ''),
|
||||
'message': warning.get('message', ''),
|
||||
'timestamp': warning.get('timestamp', '')
|
||||
}
|
||||
|
||||
# Extract metadata with line/column info
|
||||
metadata = warning.get('metadata', {})
|
||||
if metadata:
|
||||
warning_data['metadata'] = metadata
|
||||
|
||||
# Extract location info if available
|
||||
if 'line' in metadata:
|
||||
warning_data['line'] = metadata['line']
|
||||
if 'column' in metadata:
|
||||
warning_data['column'] = metadata['column']
|
||||
if 'position' in metadata:
|
||||
warning_data['position'] = metadata['position']
|
||||
if 'lineText' in metadata:
|
||||
warning_data['lineText'] = metadata['lineText']
|
||||
|
||||
# Extract examples array if present
|
||||
if 'examples' in metadata:
|
||||
warning_data['examples'] = metadata['examples']
|
||||
|
||||
processed_warnings.append(warning_data)
|
||||
|
||||
validators_data[validator_name] = {
|
||||
'valid': validator_result.get('valid', False),
|
||||
'score': validator_result.get('score', 0),
|
||||
'errorCount': validator_result.get('errorCount', 0),
|
||||
'warningCount': validator_result.get('warningCount', 0),
|
||||
'errors': processed_errors,
|
||||
'warnings': processed_warnings,
|
||||
'info': validator_result.get('info', [])
|
||||
}
|
||||
|
||||
security_lookup[key] = {
|
||||
'validated': True,
|
||||
'valid': overall.get('valid', False),
|
||||
'score': overall.get('score', 0),
|
||||
'errorCount': overall.get('errorCount', 0),
|
||||
'warningCount': overall.get('warningCount', 0),
|
||||
'lastValidated': security_data.get('timestamp', ''),
|
||||
'validators': validators_data
|
||||
}
|
||||
|
||||
# Add hash if available from integrity validator
|
||||
integrity_data = validators.get('integrity', {})
|
||||
if 'info' in integrity_data:
|
||||
for info_item in integrity_data['info']:
|
||||
# info_item is a dictionary with code, message, metadata
|
||||
if isinstance(info_item, dict):
|
||||
metadata = info_item.get('metadata', {})
|
||||
if 'fullHash' in metadata:
|
||||
security_lookup[key]['hash'] = metadata['fullHash']
|
||||
break
|
||||
elif 'hash' in metadata:
|
||||
security_lookup[key]['hash'] = metadata['hash']
|
||||
break
|
||||
|
||||
print(f"✅ Security metadata extracted for {len(security_lookup)} components")
|
||||
return security_lookup
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
print("⚠️ Security validation timed out after 5 minutes")
|
||||
return {}
|
||||
except FileNotFoundError:
|
||||
print("⚠️ npm command not found, skipping security validation")
|
||||
return {}
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error running security validation: {e}")
|
||||
return {}
|
||||
|
||||
def fetch_download_stats():
|
||||
"""
|
||||
Fetch download statistics from Supabase
|
||||
Returns a dictionary with component_type-component_name as key and download count as value
|
||||
Supports: agents, commands, mcps, settings, hooks, sandbox, skills, templates, plugins
|
||||
"""
|
||||
print("📊 Fetching download statistics from Supabase...")
|
||||
|
||||
# Define type mapping once (DRY principle)
|
||||
TYPE_MAPPING = {
|
||||
'agent': 'agents',
|
||||
'command': 'commands',
|
||||
'setting': 'settings',
|
||||
'hook': 'hooks',
|
||||
'mcp': 'mcps',
|
||||
'skill': 'skills',
|
||||
'template': 'templates',
|
||||
'plugin': 'plugins',
|
||||
'sandbox': 'sandbox'
|
||||
}
|
||||
|
||||
# Get Supabase credentials (same as generate_trending_data.py)
|
||||
supabase_url = os.getenv("SUPABASE_URL")
|
||||
supabase_api_key = os.getenv("SUPABASE_API_KEY")
|
||||
|
||||
if not supabase_url or not supabase_api_key:
|
||||
print("⚠️ Warning: Missing Supabase credentials, skipping download stats")
|
||||
return {}
|
||||
|
||||
try:
|
||||
headers = {
|
||||
'apikey': supabase_api_key,
|
||||
'Authorization': f'Bearer {supabase_api_key}'
|
||||
}
|
||||
|
||||
# First, let's query component_downloads to aggregate counts per component
|
||||
# We need to handle pagination since there can be many records
|
||||
api_url = f"{supabase_url}/rest/v1/component_downloads"
|
||||
|
||||
all_downloads = []
|
||||
offset = 0
|
||||
limit = 1000
|
||||
|
||||
# Fetch all records with pagination
|
||||
max_pages = 1000 # Safety limit (1000 pages * 1000 records = 1,000,000 max)
|
||||
for page in range(max_pages):
|
||||
paginated_headers = headers.copy()
|
||||
paginated_headers['Range'] = f'{offset}-{offset + limit - 1}'
|
||||
|
||||
response = requests.get(api_url, headers=paginated_headers)
|
||||
|
||||
if response.status_code not in [200, 206]:
|
||||
print(f" Page {page+1}: Got status {response.status_code}, stopping")
|
||||
break
|
||||
|
||||
batch = response.json()
|
||||
if not batch:
|
||||
break
|
||||
|
||||
all_downloads.extend(batch)
|
||||
|
||||
# Check if we have more records to fetch
|
||||
content_range = response.headers.get('content-range', '')
|
||||
|
||||
# If we get a range like "45000-45977/*", check if we got less than limit records
|
||||
if len(batch) < limit:
|
||||
break # We've reached the end
|
||||
|
||||
# Also check for explicit total if provided
|
||||
if content_range and '/' in content_range:
|
||||
parts = content_range.split('/')
|
||||
if parts[1] != '*':
|
||||
try:
|
||||
total = int(parts[1])
|
||||
if offset + limit >= total:
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
offset += limit
|
||||
|
||||
# Progress indicator every 10 pages
|
||||
if (page + 1) % 10 == 0:
|
||||
print(f" Fetched {len(all_downloads)} records so far...")
|
||||
|
||||
print(f"📊 Total records fetched: {len(all_downloads)}")
|
||||
|
||||
# If we fetched records, use them
|
||||
if len(all_downloads) > 0:
|
||||
# Process component_downloads data
|
||||
downloads = all_downloads
|
||||
|
||||
# Aggregate downloads by component
|
||||
download_counts = {}
|
||||
component_totals = defaultdict(int)
|
||||
|
||||
for download in downloads:
|
||||
component_type = download.get('component_type', '')
|
||||
component_name = download.get('component_name', '')
|
||||
|
||||
if component_type and component_name:
|
||||
# Handle case where component_name already includes category
|
||||
if '/' in component_name:
|
||||
category = component_name.split('/')[0]
|
||||
actual_name = component_name.split('/')[-1]
|
||||
else:
|
||||
actual_name = component_name
|
||||
category = 'general'
|
||||
|
||||
# Create a key for aggregation matching trending data structure
|
||||
key = f"{component_type}|{category}|{actual_name}"
|
||||
component_totals[key] += 1
|
||||
|
||||
# Convert to the format we need using the TYPE_MAPPING constant
|
||||
for key, count in component_totals.items():
|
||||
parts = key.split('|')
|
||||
if len(parts) == 3:
|
||||
component_type, category, component_name = parts
|
||||
mapped_type = TYPE_MAPPING.get(component_type, component_type + 's')
|
||||
final_key = f"{mapped_type}/{category}/{component_name}"
|
||||
download_counts[final_key] = count
|
||||
|
||||
print(f"✅ Fetched and aggregated {len(download_counts)} component download stats")
|
||||
return download_counts
|
||||
else:
|
||||
# Try alternative: fetch from download_stats table if it exists
|
||||
print("⚠️ No data from component_downloads, trying download_stats table...")
|
||||
alt_url = f"{supabase_url}/rest/v1/download_stats"
|
||||
alt_response = requests.get(alt_url, headers=headers)
|
||||
|
||||
if alt_response.status_code == 200:
|
||||
print("📊 Using download_stats table instead...")
|
||||
stats = alt_response.json()
|
||||
download_counts = {}
|
||||
|
||||
for stat in stats:
|
||||
component_type = stat.get('component_type', '')
|
||||
component_name = stat.get('component_name', '')
|
||||
total_downloads = stat.get('total_downloads', 0)
|
||||
|
||||
# Handle case where component_name already includes category
|
||||
if '/' in component_name:
|
||||
category = component_name.split('/')[0]
|
||||
actual_name = component_name.split('/')[-1]
|
||||
else:
|
||||
actual_name = component_name
|
||||
category = 'general'
|
||||
|
||||
# Map to plural form using TYPE_MAPPING constant
|
||||
mapped_type = TYPE_MAPPING.get(component_type, component_type + 's')
|
||||
key = f"{mapped_type}/{category}/{actual_name}"
|
||||
download_counts[key] = total_downloads
|
||||
|
||||
print(f"✅ Fetched stats for {len(download_counts)} components from download_stats")
|
||||
return download_counts
|
||||
else:
|
||||
print("⚠️ No download stats available")
|
||||
return {}
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error fetching download stats: {e}")
|
||||
return {}
|
||||
|
||||
def scan_directory_recursively(directory_path, relative_to_path=None):
|
||||
"""
|
||||
Recursively scan a directory and return all files with their relative paths.
|
||||
"""
|
||||
files_list = []
|
||||
|
||||
if not os.path.isdir(directory_path):
|
||||
return files_list
|
||||
|
||||
for root, dirs, files in os.walk(directory_path):
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
if relative_to_path:
|
||||
relative_path = os.path.relpath(file_path, relative_to_path)
|
||||
files_list.append(relative_path.replace("\\", "/"))
|
||||
else:
|
||||
files_list.append(file)
|
||||
|
||||
return files_list
|
||||
|
||||
def generate_components_json():
|
||||
"""
|
||||
Scans the cli-tool/components and cli-tool/templates directories and generates a components.json file
|
||||
for the static website, including the content of each file and download statistics.
|
||||
"""
|
||||
components_base_path = 'cli-tool/components'
|
||||
templates_base_path = 'cli-tool/templates'
|
||||
plugins_path = '.claude-plugin/marketplace.json'
|
||||
output_path = 'docs/components.json'
|
||||
components_data = {'agents': [], 'commands': [], 'mcps': [], 'settings': [], 'hooks': [], 'sandbox': [], 'skills': [], 'loops': [], 'templates': [], 'plugins': []}
|
||||
|
||||
# Run security validation
|
||||
security_metadata = run_security_validation()
|
||||
|
||||
# Fetch download statistics
|
||||
download_stats = fetch_download_stats()
|
||||
component_types = ['agents', 'commands', 'mcps', 'settings', 'hooks', 'sandbox', 'skills', 'loops']
|
||||
|
||||
print(f"Starting scan of {components_base_path} and {templates_base_path}...")
|
||||
|
||||
# Scan components (existing logic)
|
||||
for component_type in component_types:
|
||||
type_path = os.path.join(components_base_path, component_type)
|
||||
if not os.path.isdir(type_path):
|
||||
print(f"Warning: Directory not found for type: {component_type}")
|
||||
continue
|
||||
|
||||
print(f"Scanning for {component_type} in {type_path}...")
|
||||
|
||||
# Special handling for skills - they use SKILL.md inside category/skill directories
|
||||
if component_type == 'skills':
|
||||
for category in os.listdir(type_path):
|
||||
category_path = os.path.join(type_path, category)
|
||||
if os.path.isdir(category_path) and not category.endswith('.md'):
|
||||
for skill_dir in os.listdir(category_path):
|
||||
skill_dir_path = os.path.join(category_path, skill_dir)
|
||||
if os.path.isdir(skill_dir_path):
|
||||
# Look for SKILL.md inside the skill directory
|
||||
skill_file_path = os.path.join(skill_dir_path, 'SKILL.md')
|
||||
if os.path.isfile(skill_file_path):
|
||||
name = skill_dir # Use directory name as skill name
|
||||
|
||||
# Read file content
|
||||
content = ''
|
||||
description = ''
|
||||
try:
|
||||
with open(skill_file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Extract metadata from frontmatter if available
|
||||
author = ''
|
||||
repo = ''
|
||||
version = ''
|
||||
license_field = ''
|
||||
keywords = []
|
||||
if content.startswith('---'):
|
||||
frontmatter_end = content.find('---', 3)
|
||||
if frontmatter_end != -1:
|
||||
frontmatter = content[3:frontmatter_end]
|
||||
for line in frontmatter.split('\n'):
|
||||
if line.startswith('description:'):
|
||||
description = line.split('description:', 1)[1].strip()
|
||||
elif line.startswith('author:'):
|
||||
author = line.split('author:', 1)[1].strip()
|
||||
elif line.startswith('repo:'):
|
||||
repo = line.split('repo:', 1)[1].strip()
|
||||
elif line.startswith('version:'):
|
||||
version = line.split('version:', 1)[1].strip()
|
||||
elif line.startswith('license:'):
|
||||
license_field = line.split('license:', 1)[1].strip()
|
||||
elif line.startswith('tags:'):
|
||||
tags_str = line.split('tags:', 1)[1].strip()
|
||||
# Parse tags array [tag1, tag2, tag3]
|
||||
if tags_str.startswith('[') and tags_str.endswith(']'):
|
||||
tags_str = tags_str[1:-1]
|
||||
keywords = [tag.strip() for tag in tags_str.split(',')]
|
||||
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read file {skill_file_path}: {e}")
|
||||
|
||||
# Look up download count for this skill
|
||||
download_key = f"{component_type}/{category}/{name}"
|
||||
downloads = download_stats.get(download_key, 0)
|
||||
|
||||
# Look up security metadata for this skill
|
||||
security_key = f"{component_type}/{category}/{name}"
|
||||
security = security_metadata.get(security_key, {
|
||||
'validated': False,
|
||||
'valid': None,
|
||||
'score': None,
|
||||
'errorCount': 0,
|
||||
'warningCount': 0,
|
||||
'lastValidated': None
|
||||
})
|
||||
|
||||
# Collect reference files (all files except SKILL.md)
|
||||
references = []
|
||||
for ref_file in scan_directory_recursively(skill_dir_path, skill_dir_path):
|
||||
if ref_file != 'SKILL.md':
|
||||
references.append(ref_file)
|
||||
references.sort()
|
||||
|
||||
# Path includes category for proper organization
|
||||
component = {
|
||||
'name': name, # Just the skill directory name
|
||||
'path': f"{category}/{name}", # category/skill-name format
|
||||
'category': category,
|
||||
'type': 'skill',
|
||||
'content': content,
|
||||
'description': description,
|
||||
'author': author,
|
||||
'repo': repo,
|
||||
'version': version,
|
||||
'license': license_field,
|
||||
'keywords': keywords,
|
||||
'downloads': downloads,
|
||||
'security': security,
|
||||
'references': references
|
||||
}
|
||||
components_data[component_type].append(component)
|
||||
print(f" Processed skill: {category}/{name}")
|
||||
continue # Skip the normal file scanning for skills
|
||||
|
||||
# Normal scanning for other component types
|
||||
for category in os.listdir(type_path):
|
||||
category_path = os.path.join(type_path, category)
|
||||
if os.path.isdir(category_path):
|
||||
for file_name in os.listdir(category_path):
|
||||
file_path = os.path.join(category_path, file_name)
|
||||
if os.path.isfile(file_path) and (file_name.endswith('.md') or file_name.endswith('.json')) and not file_name.endswith('.py'):
|
||||
name = os.path.splitext(file_name)[0]
|
||||
|
||||
# Read file content
|
||||
content = ''
|
||||
description = ''
|
||||
author = ''
|
||||
repo = ''
|
||||
version = ''
|
||||
license_field = ''
|
||||
keywords = []
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Extract description, author and repo from JSON files
|
||||
if file_name.endswith('.json'):
|
||||
try:
|
||||
import json
|
||||
json_data = json.loads(content)
|
||||
|
||||
if component_type == 'mcps':
|
||||
# Extract description from the first mcpServer entry
|
||||
if 'mcpServers' in json_data:
|
||||
for server_name, server_config in json_data['mcpServers'].items():
|
||||
if isinstance(server_config, dict) and 'description' in server_config:
|
||||
description = server_config['description']
|
||||
break # Use the first description found
|
||||
elif component_type in ['settings', 'hooks']:
|
||||
# Extract metadata from settings/hooks JSON files
|
||||
if 'description' in json_data:
|
||||
description = json_data['description']
|
||||
if 'author' in json_data:
|
||||
author = json_data['author']
|
||||
if 'repo' in json_data:
|
||||
repo = json_data['repo']
|
||||
if 'version' in json_data:
|
||||
version = json_data['version']
|
||||
if 'license' in json_data:
|
||||
license_field = json_data['license']
|
||||
if 'keywords' in json_data and isinstance(json_data['keywords'], list):
|
||||
keywords = json_data['keywords']
|
||||
|
||||
except json.JSONDecodeError:
|
||||
print(f"Warning: Invalid JSON in {file_path}")
|
||||
|
||||
# Extract metadata from markdown frontmatter
|
||||
elif file_name.endswith('.md'):
|
||||
if content.startswith('---'):
|
||||
frontmatter_end = content.find('---', 3)
|
||||
if frontmatter_end != -1:
|
||||
frontmatter = content[3:frontmatter_end]
|
||||
for line in frontmatter.split('\n'):
|
||||
if line.startswith('description:'):
|
||||
description = line.split('description:', 1)[1].strip()
|
||||
elif line.startswith('author:'):
|
||||
author = line.split('author:', 1)[1].strip()
|
||||
elif line.startswith('repo:'):
|
||||
repo = line.split('repo:', 1)[1].strip()
|
||||
elif line.startswith('version:'):
|
||||
version = line.split('version:', 1)[1].strip()
|
||||
elif line.startswith('license:'):
|
||||
license_field = line.split('license:', 1)[1].strip()
|
||||
elif line.startswith('tags:'):
|
||||
tags_str = line.split('tags:', 1)[1].strip()
|
||||
# Parse tags array [tag1, tag2, tag3]
|
||||
if tags_str.startswith('[') and tags_str.endswith(']'):
|
||||
tags_str = tags_str[1:-1]
|
||||
keywords = [tag.strip() for tag in tags_str.split(',')]
|
||||
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read file {file_path}: {e}")
|
||||
|
||||
# Look up download count for this component
|
||||
# The key in download_stats uses the plural form: agents/category/name
|
||||
download_key = f"{component_type}/{category}/{name}"
|
||||
downloads = download_stats.get(download_key, 0)
|
||||
|
||||
# Look up security metadata for this component
|
||||
security_key = f"{component_type}/{category}/{name}"
|
||||
security = security_metadata.get(security_key, {
|
||||
'validated': False,
|
||||
'valid': None,
|
||||
'score': None,
|
||||
'errorCount': 0,
|
||||
'warningCount': 0,
|
||||
'lastValidated': None
|
||||
})
|
||||
|
||||
component = {
|
||||
'name': name,
|
||||
'path': os.path.join(category, file_name).replace("\\", "/"),
|
||||
'category': category,
|
||||
'type': component_type[:-1], # singular form
|
||||
'content': content, # Add file content
|
||||
'description': description, # Add description for MCPs
|
||||
'author': author, # Add author field
|
||||
'repo': repo, # Add repository field
|
||||
'version': version, # Add version field
|
||||
'license': license_field, # Add license field
|
||||
'keywords': keywords, # Add keywords field
|
||||
'downloads': downloads, # Add download count
|
||||
'security': security # Add security metadata
|
||||
}
|
||||
components_data[component_type].append(component)
|
||||
|
||||
# Scan templates (new logic)
|
||||
if os.path.isdir(templates_base_path):
|
||||
print(f"Scanning for templates in {templates_base_path}...")
|
||||
|
||||
for language_dir in os.listdir(templates_base_path):
|
||||
language_path = os.path.join(templates_base_path, language_dir)
|
||||
if os.path.isdir(language_path):
|
||||
print(f" Processing language: {language_dir}")
|
||||
|
||||
# Collect files in the language directory (excluding examples folder)
|
||||
language_files = []
|
||||
for item in os.listdir(language_path):
|
||||
item_path = os.path.join(language_path, item)
|
||||
if os.path.isfile(item_path):
|
||||
language_files.append(item)
|
||||
elif os.path.isdir(item_path) and item != 'examples':
|
||||
# For directories like .claude, scan recursively
|
||||
if item == '.claude':
|
||||
recursive_files = scan_directory_recursively(item_path, language_path)
|
||||
language_files.extend(recursive_files)
|
||||
else:
|
||||
# For other directories, just add the directory name (optional)
|
||||
pass
|
||||
|
||||
# Look up download count for this template
|
||||
template_download_key = f"templates/{language_dir}"
|
||||
template_downloads = download_stats.get(template_download_key, 0)
|
||||
|
||||
# Create language template entry
|
||||
language_template = {
|
||||
'name': language_dir,
|
||||
'id': language_dir,
|
||||
'type': 'template',
|
||||
'subtype': 'language',
|
||||
'category': 'languages',
|
||||
'description': f'{language_dir.title()} project template',
|
||||
'files': language_files,
|
||||
'installCommand': f'npx claude-code-templates@latest --template={language_dir} --yes',
|
||||
'downloads': template_downloads
|
||||
}
|
||||
components_data['templates'].append(language_template)
|
||||
|
||||
# Check for examples folder (contains frameworks)
|
||||
examples_path = os.path.join(language_path, 'examples')
|
||||
if os.path.isdir(examples_path):
|
||||
for framework_dir in os.listdir(examples_path):
|
||||
framework_path = os.path.join(examples_path, framework_dir)
|
||||
if os.path.isdir(framework_path):
|
||||
print(f" Processing framework: {framework_dir}")
|
||||
|
||||
# Collect files in the framework directory
|
||||
framework_files = []
|
||||
for item in os.listdir(framework_path):
|
||||
item_path = os.path.join(framework_path, item)
|
||||
if os.path.isfile(item_path):
|
||||
framework_files.append(item)
|
||||
elif os.path.isdir(item_path):
|
||||
# For directories like .claude, scan recursively
|
||||
if item == '.claude':
|
||||
recursive_files = scan_directory_recursively(item_path, framework_path)
|
||||
framework_files.extend(recursive_files)
|
||||
else:
|
||||
# For other directories, just add the directory name (optional)
|
||||
pass
|
||||
|
||||
# Look up download count for this framework
|
||||
framework_download_key = f"templates/{framework_dir}"
|
||||
framework_downloads = download_stats.get(framework_download_key, 0)
|
||||
|
||||
# Create framework template entry
|
||||
framework_template = {
|
||||
'name': framework_dir,
|
||||
'id': framework_dir,
|
||||
'type': 'template',
|
||||
'subtype': 'framework',
|
||||
'category': 'frameworks',
|
||||
'language': language_dir,
|
||||
'description': f'{framework_dir.title()} with {language_dir.title()}',
|
||||
'files': framework_files,
|
||||
'installCommand': f'npx claude-code-templates@latest --template={framework_dir} --yes',
|
||||
'downloads': framework_downloads
|
||||
}
|
||||
components_data['templates'].append(framework_template)
|
||||
else:
|
||||
print(f"Warning: Templates directory not found: {templates_base_path}")
|
||||
|
||||
# Load components metadata from marketplace.json (Claude Code standard)
|
||||
components_marketplace_path = 'cli-tool/components/.claude-plugin/marketplace.json'
|
||||
components_marketplace = None
|
||||
if os.path.isfile(components_marketplace_path):
|
||||
print(f"Loading components metadata from {components_marketplace_path}...")
|
||||
try:
|
||||
with open(components_marketplace_path, 'r', encoding='utf-8') as f:
|
||||
components_marketplace = json.load(f)
|
||||
print(f"✅ Loaded metadata for {len(components_marketplace.get('agents', []))} agents")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error reading components metadata from {components_marketplace_path}: {e}")
|
||||
|
||||
# Scan plugins from marketplace.json and store marketplace data
|
||||
marketplace_full_data = None
|
||||
if os.path.isfile(plugins_path):
|
||||
print(f"Scanning for plugins in {plugins_path}...")
|
||||
try:
|
||||
with open(plugins_path, 'r', encoding='utf-8') as f:
|
||||
marketplace_data = json.load(f)
|
||||
|
||||
# Store full marketplace data for inclusion in components.json
|
||||
marketplace_full_data = marketplace_data
|
||||
|
||||
if 'plugins' in marketplace_data:
|
||||
for plugin in marketplace_data['plugins']:
|
||||
plugin_name = plugin.get('name', '')
|
||||
plugin_description = plugin.get('description', '')
|
||||
plugin_version = plugin.get('version', '1.0.0')
|
||||
plugin_keywords = plugin.get('keywords', [])
|
||||
plugin_author = plugin.get('author', {}).get('name', '')
|
||||
|
||||
# Get component lists with file names
|
||||
commands_list = plugin.get('commands', [])
|
||||
agents_list = plugin.get('agents', [])
|
||||
mcps_list = plugin.get('mcpServers', [])
|
||||
|
||||
# Extract component names with category/folder from file paths
|
||||
def extract_component_with_category(path):
|
||||
# Extract category/name from path like "./cli-tool/components/commands/git/feature.md"
|
||||
# Result should be "git/feature"
|
||||
parts = path.split('/')
|
||||
if len(parts) >= 2:
|
||||
filename = parts[-1].replace('.md', '').replace('.json', '')
|
||||
category = parts[-2]
|
||||
return f"{category}/{filename}"
|
||||
else:
|
||||
# Fallback to just filename if path structure is unexpected
|
||||
filename = path.split('/')[-1]
|
||||
return filename.replace('.md', '').replace('.json', '')
|
||||
|
||||
commands_names = [extract_component_with_category(cmd) for cmd in commands_list]
|
||||
agents_names = [extract_component_with_category(agent) for agent in agents_list]
|
||||
mcps_names = [extract_component_with_category(mcp) for mcp in mcps_list]
|
||||
|
||||
# Look up download count for this plugin
|
||||
plugin_download_key = f"plugins/{plugin_name}"
|
||||
plugin_downloads = download_stats.get(plugin_download_key, 0)
|
||||
|
||||
# Create plugin entry
|
||||
plugin_entry = {
|
||||
'name': plugin_name,
|
||||
'id': plugin_name,
|
||||
'type': 'plugin',
|
||||
'description': plugin_description,
|
||||
'version': plugin_version,
|
||||
'keywords': plugin_keywords,
|
||||
'author': plugin_author,
|
||||
'commands': len(commands_list),
|
||||
'agents': len(agents_list),
|
||||
'mcpServers': len(mcps_list),
|
||||
'commandsList': commands_names,
|
||||
'agentsList': agents_names,
|
||||
'mcpServersList': mcps_names,
|
||||
'installCommand': f'/plugin install {plugin_name}@claude-code-templates',
|
||||
'downloads': plugin_downloads
|
||||
}
|
||||
components_data['plugins'].append(plugin_entry)
|
||||
print(f" Processed plugin: {plugin_name}")
|
||||
|
||||
print(f"✅ Processed {len(components_data['plugins'])} plugins from marketplace.json")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error reading plugins from {plugins_path}: {e}")
|
||||
else:
|
||||
print(f"Warning: Plugins file not found: {plugins_path}")
|
||||
|
||||
# Sort components alphabetically
|
||||
for component_type in components_data:
|
||||
if component_type in ['templates', 'plugins']:
|
||||
# Sort templates and plugins by name since they don't have path
|
||||
components_data[component_type].sort(key=lambda x: x['name'])
|
||||
else:
|
||||
# Sort other components by path
|
||||
components_data[component_type].sort(key=lambda x: x['path'])
|
||||
|
||||
# Add marketplace metadata (root level - our public marketplace)
|
||||
if marketplace_full_data:
|
||||
components_data['marketplace'] = marketplace_full_data
|
||||
print("✅ Added public marketplace metadata to components.json")
|
||||
|
||||
# Add components marketplace metadata (Claude Code standard)
|
||||
if components_marketplace:
|
||||
components_data['componentsMarketplace'] = components_marketplace
|
||||
print("✅ Added components marketplace metadata to components.json")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Performance optimization: separate content from the lightweight index.
|
||||
#
|
||||
# docs/components.json → index without 'content' (~87% smaller)
|
||||
# dashboard/public/components.json → same, served by Cloudflare Pages
|
||||
# dashboard/public/component-content/{type}/{slug}.json → per-component
|
||||
# raw content, loaded on demand only on detail pages.
|
||||
# ---------------------------------------------------------------------------
|
||||
dashboard_public_dir = 'dashboard/public'
|
||||
dashboard_content_dir = os.path.join(dashboard_public_dir, 'component-content')
|
||||
|
||||
# Component types that carry a 'content' field
|
||||
content_bearing_types = ['agents', 'commands', 'mcps', 'settings', 'hooks', 'sandbox', 'skills', 'loops']
|
||||
|
||||
# 1. Write per-component content files directly to dashboard/public/component-content/
|
||||
os.makedirs(dashboard_content_dir, exist_ok=True)
|
||||
written_content_count = 0
|
||||
for ctype in content_bearing_types:
|
||||
for component in components_data.get(ctype, []):
|
||||
raw_content = component.get('content', '')
|
||||
if not raw_content:
|
||||
continue
|
||||
# Slug: path with extension stripped (matches detail-page URL lookup)
|
||||
slug = component['path'].replace('.md', '').replace('.json', '')
|
||||
content_file_path = os.path.join(dashboard_content_dir, ctype, slug + '.json')
|
||||
os.makedirs(os.path.dirname(content_file_path), exist_ok=True)
|
||||
try:
|
||||
with open(content_file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump({'content': raw_content}, f, ensure_ascii=False)
|
||||
written_content_count += 1
|
||||
except IOError as e:
|
||||
print(f"Warning: Could not write content file {content_file_path}: {e}")
|
||||
|
||||
print(f"Generated {written_content_count} component content files in {dashboard_content_dir}/")
|
||||
|
||||
# 2. Build the legacy/docs index: strip only 'content'.
|
||||
# The legacy static site (docs/js/*) renders security badges, so the
|
||||
# 'security' field MUST stay in docs/components.json.
|
||||
index_data = {}
|
||||
for k, v in components_data.items():
|
||||
if k in content_bearing_types:
|
||||
index_data[k] = [{key: val for key, val in c.items() if key != 'content'} for c in v]
|
||||
else:
|
||||
index_data[k] = v
|
||||
|
||||
# 3. Write the docs index (with 'security', without 'content').
|
||||
try:
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(index_data, f, indent=2, ensure_ascii=False)
|
||||
print(f"Successfully generated {output_path} (index without content).")
|
||||
except IOError as e:
|
||||
print(f"Error writing to {output_path}: {e}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dashboard-specific artifacts (Cloudflare Pages, www.aitmpl.com).
|
||||
#
|
||||
# The dashboard never reads the 'security' field, so we strip it here to
|
||||
# shrink the payload the browser downloads. We also emit smaller,
|
||||
# purpose-built artifacts so islands stop downloading the whole index:
|
||||
#
|
||||
# components.json → full index, no 'content', no 'security'
|
||||
# counts.json → { type: count } (sidebars/plugins pages)
|
||||
# components/{type}.json → per-type slice (grid loads only active type)
|
||||
# search-index.json → flat [{type,name,path,description,category}]
|
||||
# ---------------------------------------------------------------------------
|
||||
STRIP_KEYS = {'content', 'security'}
|
||||
|
||||
def strip_for_dashboard(component):
|
||||
return {key: val for key, val in component.items() if key not in STRIP_KEYS}
|
||||
|
||||
# Build the dashboard index (content + security stripped from components).
|
||||
dashboard_index_data = {}
|
||||
for k, v in index_data.items():
|
||||
if k in content_bearing_types and isinstance(v, list):
|
||||
dashboard_index_data[k] = [strip_for_dashboard(c) for c in v]
|
||||
elif isinstance(v, list):
|
||||
# Non-content types (e.g. mcps/settings/hooks may carry security)
|
||||
dashboard_index_data[k] = [strip_for_dashboard(c) for c in v]
|
||||
else:
|
||||
dashboard_index_data[k] = v
|
||||
|
||||
try:
|
||||
os.makedirs(dashboard_public_dir, exist_ok=True)
|
||||
dashboard_index_path = os.path.join(dashboard_public_dir, 'components.json')
|
||||
with open(dashboard_index_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(dashboard_index_data, f, ensure_ascii=False)
|
||||
print(f"Generated {dashboard_index_path} (dashboard index, no content/security).")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not write dashboard index: {e}")
|
||||
|
||||
# counts.json — tiny per-type counts so sidebars/plugins pages stop
|
||||
# downloading the full index just to read array lengths.
|
||||
try:
|
||||
counts = {k: len(v) for k, v in dashboard_index_data.items() if isinstance(v, list)}
|
||||
counts_path = os.path.join(dashboard_public_dir, 'counts.json')
|
||||
with open(counts_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(counts, f, ensure_ascii=False)
|
||||
print(f"Generated {counts_path} ({counts}).")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not write counts.json: {e}")
|
||||
|
||||
# components/{type}.json — per-type slices so the grid loads only the
|
||||
# active type instead of all 1800+ components at once.
|
||||
try:
|
||||
per_type_dir = os.path.join(dashboard_public_dir, 'components')
|
||||
os.makedirs(per_type_dir, exist_ok=True)
|
||||
for k, v in dashboard_index_data.items():
|
||||
if not isinstance(v, list):
|
||||
continue
|
||||
type_path = os.path.join(per_type_dir, f'{k}.json')
|
||||
with open(type_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(v, f, ensure_ascii=False)
|
||||
print(f"Generated per-type index files in {per_type_dir}/")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not write per-type index files: {e}")
|
||||
|
||||
# search-index.json — flat array with only the fields the global (Cmd+K)
|
||||
# search reads: type, name, path, description, category.
|
||||
try:
|
||||
search_index = []
|
||||
for k, v in dashboard_index_data.items():
|
||||
if not isinstance(v, list):
|
||||
continue
|
||||
for c in v:
|
||||
search_index.append({
|
||||
'type': c.get('type'),
|
||||
'name': c.get('name'),
|
||||
'path': c.get('path'),
|
||||
'description': c.get('description', ''),
|
||||
'category': c.get('category'),
|
||||
})
|
||||
search_path = os.path.join(dashboard_public_dir, 'search-index.json')
|
||||
with open(search_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(search_index, f, ensure_ascii=False)
|
||||
print(f"Generated {search_path} ({len(search_index)} entries).")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not write search-index.json: {e}")
|
||||
|
||||
# Log summary (uses original components_data which still has content for accurate counts)
|
||||
print("\n--- Generation Summary ---")
|
||||
for component_type, components in components_data.items():
|
||||
# Skip marketplace metadata in summary (it's not a component type)
|
||||
if component_type in ['marketplace', 'componentsMarketplace']:
|
||||
continue
|
||||
|
||||
print(f" - Found and processed {len(components)} {component_type}")
|
||||
if component_type == 'templates':
|
||||
languages = len([t for t in components if t.get('subtype') == 'language'])
|
||||
frameworks = len([t for t in components if t.get('subtype') == 'framework'])
|
||||
print(f" • {languages} languages, {frameworks} frameworks")
|
||||
elif component_type == 'plugins':
|
||||
total_commands = sum(p.get('commands', 0) for p in components)
|
||||
total_agents = sum(p.get('agents', 0) for p in components)
|
||||
total_mcps = sum(p.get('mcpServers', 0) for p in components)
|
||||
print(f" • {total_commands} commands, {total_agents} agents, {total_mcps} MCPs")
|
||||
|
||||
# Security statistics for components with security metadata
|
||||
if component_type not in ['templates', 'plugins']:
|
||||
validated = len([c for c in components if c.get('security', {}).get('validated', False)])
|
||||
valid_components = len([c for c in components if c.get('security', {}).get('valid', False)])
|
||||
avg_score = sum(c.get('security', {}).get('score', 0) for c in components if c.get('security', {}).get('validated', False)) / validated if validated > 0 else 0
|
||||
print(f" • Security: {validated} validated, {valid_components} passed, avg score: {avg_score:.1f}")
|
||||
|
||||
print("--------------------------")
|
||||
|
||||
if __name__ == '__main__':
|
||||
generate_components_json()
|
||||
Reference in New Issue
Block a user