chore: import upstream snapshot with attribution

utils/add_tags.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+import random
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+
+from maigret.maigret import MaigretDatabase
+
+
+def update_tags(site):
+    tags = []
+    if not site.tags:
+        print(f'Site {site.name} doesn\'t have tags')
+    else:
+        tags = site.tags
+        print(f'Site {site.name} tags: ' + ', '.join(tags))
+
+    print(f'URL: {site.url_main}')
+
+    new_tags = set(input('Enter new tags: ').split(', '))
+    if "disabled" in new_tags:
+        new_tags.remove("disabled")
+        site.disabled = True
+
+    site.tags = [x for x in list(new_tags) if x]
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter
+                            )
+    parser.add_argument("--base","-b", metavar="BASE_FILE",
+                        dest="base_file", default="maigret/resources/data.json",
+                        help="JSON file with sites data to update.")
+    parser.add_argument("--name", help="Name of site to check")
+
+    pool = list()
+
+    args = parser.parse_args()
+
+    db = MaigretDatabase()
+    db.load_from_file(args.base_file).sites
+
+    while True:
+        if args.name:
+            sites = list(db.ranked_sites_dict(names=[args.name]).values())
+            site = random.choice(sites)
+        else:
+            site = random.choice(db.sites)
+
+        if site.engine == 'uCoz':
+            continue
+
+        # if not 'in' in site.tags:
+        #     continue
+
+        update_tags(site)
+
+        db.save_to_file(args.base_file)

utils/check_engines.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Maigret: Supported Site Listing with Alexa ranking and country tags
+This module generates the listing of supported sites in file `SITES.md`
+and pretty prints file with sites data.
+"""
+import asyncio
+import json
+import logging
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+
+from maigret.maigret import get_response
+from maigret.sites import MaigretDatabase, MaigretEngine
+
+async def check_engine_of_site(site_name, sites_with_engines, future, engine_name, semaphore, logger):
+    async with semaphore:
+        response = await get_response(request_future=future,
+                                      site_name=site_name,
+                                      logger=logger)
+
+        html_text, status_code, error_text, expection_text = response
+
+        if html_text and engine_name in html_text:
+            sites_with_engines.append(site_name)
+            return True
+    return False
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter
+                            )
+    parser.add_argument("--base","-b", metavar="BASE_FILE",
+                        dest="base_file", default="maigret/resources/data.json",
+                        help="JSON file with sites data to update.")
+
+    parser.add_argument('--engine', '-e', help='check only selected engine', type=str)
+
+    args = parser.parse_args()
+
+    log_level = logging.INFO
+    logging.basicConfig(
+        format='[%(filename)s:%(lineno)d] %(levelname)-3s  %(asctime)s %(message)s',
+        datefmt='%H:%M:%S',
+        level=log_level
+    )
+    logger = logging.getLogger('engines-check')
+    logger.setLevel(log_level)
+
+    db = MaigretDatabase()
+    sites_subset = db.load_from_file(args.base_file).sites
+    sites = {site.name: site for site in sites_subset}
+
+    with open(args.base_file, "r", encoding="utf-8") as data_file:
+        sites_info = json.load(data_file)
+        engines = sites_info['engines']
+
+    for engine_name, engine_data in engines.items():
+        if args.engine and args.engine != engine_name:
+            continue
+
+        if not 'presenseStrs' in engine_data:
+            print(f'No features to automatically detect sites on engine {engine_name}')
+            continue
+
+        engine_obj = MaigretEngine(engine_name, engine_data)
+
+        # setup connections for checking both engine and usernames
+        connector = aiohttp.TCPConnector(ssl=False)
+        connector.verify_ssl=False
+        session = aiohttp.ClientSession(connector=connector)
+
+        sem = asyncio.Semaphore(100)
+        loop = asyncio.get_event_loop()
+        tasks = []
+
+        # check sites without engine if they look like sites on this engine
+        new_engine_sites = []
+        for site_name, site_data in sites.items():
+            if site_data.engine:
+                continue
+
+            future = session.get(url=site_data.url_main,
+                                 allow_redirects=True,
+                                 timeout=10,
+                                 )
+
+            check_engine_coro = check_engine_of_site(site_name, new_engine_sites, future, engine_name, sem, logger)
+            future = asyncio.ensure_future(check_engine_coro)
+            tasks.append(future)
+
+        # progress bar
+        with alive_progress(len(tasks), title='Checking sites') as progress:
+            for f in asyncio.as_completed(tasks):
+                loop.run_until_complete(f)
+                progress()
+
+        print(f'Total detected {len(new_engine_sites)} sites on engine {engine_name}')
+        # dict with new found engine sites
+        new_sites = {site_name: sites[site_name] for site_name in new_engine_sites}
+
+        # update sites obj from engine
+        for site_name, site in new_sites.items():
+            site.request_future = None
+            site.engine = engine_name
+            site.update_from_engine(engine_obj)
+
+        async def update_site_data(site_name, site_data, all_sites, logger, no_progressbar):
+            updates = await site_self_check(site_name, site_data, logger, no_progressbar)
+            all_sites[site_name].update(updates)
+
+        tasks = []
+        # for new_site_name, new_site_data in new_sites.items():
+            # coro = update_site_data(new_site_name, new_site_data, new_sites, logger)
+            # future = asyncio.ensure_future(coro)
+            # tasks.append(future)
+
+        # asyncio.gather(*tasks)
+        for new_site_name, new_site_data in new_sites.items():
+            coro = update_site_data(new_site_name, new_site_data, new_sites, logger, no_progressbar=True)
+            loop.run_until_complete(coro)
+
+        updated_sites_count = 0
+
+        for s in new_sites:
+            site = new_sites[s]
+            site.request_future = None
+
+            if site.disabled:
+                print(f'{site.name} failed username checking of engine {engine_name}')
+                continue
+
+            site = site.strip_engine_data()
+
+            db.update_site(site)
+            updated_sites_count += 1
+            db.save_to_file(args.base_file)
+
+            print(f'Site "{s}": ' + json.dumps(site.json, indent=4))
+
+        print(f'Updated total {updated_sites_count} sites!')
+        print(f'Checking all sites on engine {engine_name}')
+
+        loop.run_until_complete(session.close())
+
+    print("\nFinished updating supported site listing!")

utils/check_top_n.py

@@ -0,0 +1,480 @@
+#!/usr/bin/env python3
+"""
+Mass site checking utility for Maigret development.
+Check top-N sites from data.json and generate a report.
+
+Usage:
+    python utils/check_top_n.py --top 100                    # Check top 100 sites
+    python utils/check_top_n.py --top 50 --parallel 10       # Check with 10 parallel requests
+    python utils/check_top_n.py --top 100 --output report.json
+    python utils/check_top_n.py --top 100 --fix              # Auto-fix simple issues
+"""
+
+import argparse
+import asyncio
+import json
+import sys
+import time
+from collections import defaultdict
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+# Add parent dir for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+try:
+    import aiohttp
+except ImportError:
+    print("aiohttp not installed. Run: pip install aiohttp")
+    sys.exit(1)
+
+
+class Colors:
+    RED = "\033[91m"
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    BLUE = "\033[94m"
+    CYAN = "\033[96m"
+    RESET = "\033[0m"
+    BOLD = "\033[1m"
+
+
+def color(text: str, c: str) -> str:
+    return f"{c}{text}{Colors.RESET}"
+
+
+@dataclass
+class SiteCheckResult:
+    """Result of checking a single site."""
+    site_name: str
+    alexa_rank: int
+    disabled: bool
+    check_type: str
+
+    # Status
+    status: str = "unknown"  # working, broken, timeout, error, anti_bot, disabled
+
+    # HTTP results
+    claimed_http_status: Optional[int] = None
+    unclaimed_http_status: Optional[int] = None
+    claimed_error: Optional[str] = None
+    unclaimed_error: Optional[str] = None
+
+    # Issues detected
+    issues: List[str] = field(default_factory=list)
+    warnings: List[str] = field(default_factory=list)
+
+    # Recommendations
+    recommendations: List[str] = field(default_factory=list)
+
+    # Timing
+    check_time_ms: int = 0
+
+
+DEFAULT_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+}
+
+
+async def check_url(url: str, headers: dict, timeout: int = 15) -> dict:
+    """Quick URL check returning status and basic info."""
+    result = {
+        "status": None,
+        "final_url": None,
+        "content_length": 0,
+        "error": None,
+        "error_type": None,
+        "content": None,
+        "markers": {},
+    }
+
+    try:
+        connector = aiohttp.TCPConnector(ssl=False)
+        timeout_obj = aiohttp.ClientTimeout(total=timeout)
+
+        async with aiohttp.ClientSession(connector=connector, timeout=timeout_obj) as session:
+            async with session.get(url, headers=headers, allow_redirects=True) as resp:
+                result["status"] = resp.status
+                result["final_url"] = str(resp.url)
+
+                try:
+                    text = await resp.text()
+                    result["content_length"] = len(text)
+                    result["content"] = text
+
+                    text_lower = text.lower()
+                    result["markers"] = {
+                        "404_text": any(m in text_lower for m in ["not found", "404", "doesn't exist"]),
+                        "captcha": any(m in text_lower for m in ["captcha", "recaptcha", "challenge"]),
+                        "cloudflare": "cloudflare" in text_lower,
+                        "login": any(m in text_lower for m in ["log in", "login", "sign in"]),
+                    }
+                except Exception as e:
+                    result["error"] = f"Content error: {e}"
+                    result["error_type"] = "content"
+
+    except asyncio.TimeoutError:
+        result["error"] = "Timeout"
+        result["error_type"] = "timeout"
+    except aiohttp.ClientError as e:
+        result["error"] = str(e)
+        result["error_type"] = "client"
+    except Exception as e:
+        result["error"] = str(e)
+        result["error_type"] = "unknown"
+
+    return result
+
+
+async def check_site(site_name: str, config: dict, timeout: int = 15) -> SiteCheckResult:
+    """Check a single site and return detailed result."""
+    start_time = time.time()
+
+    result = SiteCheckResult(
+        site_name=site_name,
+        alexa_rank=config.get("alexaRank", 999999),
+        disabled=config.get("disabled", False),
+        check_type=config.get("checkType", "status_code"),
+    )
+
+    # Skip disabled sites
+    if result.disabled:
+        result.status = "disabled"
+        return result
+
+    # Build URL
+    url_template = config.get("url", "")
+    url_main = config.get("urlMain", "")
+    url_subpath = config.get("urlSubpath", "")
+    url_template = url_template.replace("{urlMain}", url_main).replace("{urlSubpath}", url_subpath)
+
+    claimed = config.get("usernameClaimed")
+    unclaimed = config.get("usernameUnclaimed", "noonewouldeverusethis7")
+
+    if not claimed:
+        result.status = "error"
+        result.issues.append("No usernameClaimed defined")
+        return result
+
+    # Prepare headers
+    headers = DEFAULT_HEADERS.copy()
+    if config.get("headers"):
+        headers.update(config["headers"])
+
+    # Check both URLs
+    url_claimed = url_template.replace("{username}", claimed)
+    url_unclaimed = url_template.replace("{username}", unclaimed)
+
+    try:
+        claimed_result, unclaimed_result = await asyncio.gather(
+            check_url(url_claimed, headers, timeout),
+            check_url(url_unclaimed, headers, timeout),
+        )
+    except Exception as e:
+        result.status = "error"
+        result.issues.append(f"Check failed: {e}")
+        return result
+
+    result.claimed_http_status = claimed_result["status"]
+    result.unclaimed_http_status = unclaimed_result["status"]
+    result.claimed_error = claimed_result.get("error")
+    result.unclaimed_error = unclaimed_result.get("error")
+
+    # Categorize result
+    if claimed_result["error_type"] == "timeout" or unclaimed_result["error_type"] == "timeout":
+        result.status = "timeout"
+        result.issues.append("Request timeout")
+
+    elif claimed_result["status"] == 403 or claimed_result["status"] == 429:
+        result.status = "anti_bot"
+        result.issues.append(f"Anti-bot protection (HTTP {claimed_result['status']})")
+
+    elif claimed_result.get("markers", {}).get("captcha"):
+        result.status = "anti_bot"
+        result.issues.append("Captcha detected")
+
+    elif claimed_result.get("markers", {}).get("cloudflare"):
+        result.status = "anti_bot"
+        result.warnings.append("Cloudflare protection detected")
+
+    elif claimed_result["error"] or unclaimed_result["error"]:
+        result.status = "error"
+        if claimed_result["error"]:
+            result.issues.append(f"Claimed error: {claimed_result['error']}")
+        if unclaimed_result["error"]:
+            result.issues.append(f"Unclaimed error: {unclaimed_result['error']}")
+
+    else:
+        # Validate check type
+        check_type = config.get("checkType", "status_code")
+
+        if check_type == "status_code":
+            if claimed_result["status"] == unclaimed_result["status"]:
+                result.status = "broken"
+                result.issues.append(f"Same status code ({claimed_result['status']}) for both")
+                # Suggest fix
+                if claimed_result["final_url"] != unclaimed_result["final_url"]:
+                    result.recommendations.append("Switch to checkType: response_url")
+            else:
+                result.status = "working"
+
+        elif check_type == "response_url":
+            if claimed_result["final_url"] == unclaimed_result["final_url"]:
+                result.status = "broken"
+                result.issues.append("Same final URL for both")
+                if claimed_result["status"] != unclaimed_result["status"]:
+                    result.recommendations.append("Switch to checkType: status_code")
+            else:
+                result.status = "working"
+
+        elif check_type == "message":
+            presense_strs = config.get("presenseStrs", [])
+            absence_strs = config.get("absenceStrs", [])
+
+            claimed_content = claimed_result.get("content", "") or ""
+            unclaimed_content = unclaimed_result.get("content", "") or ""
+
+            presense_ok = not presense_strs or any(s in claimed_content for s in presense_strs)
+            absence_claimed = absence_strs and any(s in claimed_content for s in absence_strs)
+            absence_unclaimed = absence_strs and any(s in unclaimed_content for s in absence_strs)
+
+            if presense_strs and not presense_ok:
+                result.status = "broken"
+                result.issues.append(f"presenseStrs not found: {presense_strs}")
+                # Check if status_code would work
+                if claimed_result["status"] != unclaimed_result["status"]:
+                    result.recommendations.append(f"Switch to checkType: status_code ({claimed_result['status']} vs {unclaimed_result['status']})")
+            elif absence_claimed:
+                result.status = "broken"
+                result.issues.append(f"absenceStrs found in claimed page")
+            elif absence_strs and not absence_unclaimed:
+                result.status = "broken"
+                result.warnings.append("absenceStrs not found in unclaimed page")
+            else:
+                result.status = "working"
+
+        else:
+            result.status = "unknown"
+            result.warnings.append(f"Unknown checkType: {check_type}")
+
+    result.check_time_ms = int((time.time() - start_time) * 1000)
+    return result
+
+
+def load_sites(db_path: Path) -> Dict[str, dict]:
+    """Load all sites from data.json."""
+    with open(db_path) as f:
+        data = json.load(f)
+    return data.get("sites", {})
+
+
+def get_top_sites(sites: Dict[str, dict], n: int) -> List[Tuple[str, dict]]:
+    """Get top N sites by Alexa rank."""
+    ranked = []
+    for name, config in sites.items():
+        rank = config.get("alexaRank", 999999)
+        ranked.append((name, config, rank))
+
+    ranked.sort(key=lambda x: x[2])
+    return [(name, config) for name, config, _ in ranked[:n]]
+
+
+async def check_sites_batch(sites: List[Tuple[str, dict]], parallel: int = 5,
+                            timeout: int = 15, progress_callback=None) -> List[SiteCheckResult]:
+    """Check multiple sites with parallelism control."""
+    results = []
+    semaphore = asyncio.Semaphore(parallel)
+
+    async def check_with_semaphore(name, config, index):
+        async with semaphore:
+            if progress_callback:
+                progress_callback(index, len(sites), name)
+            return await check_site(name, config, timeout)
+
+    tasks = [
+        check_with_semaphore(name, config, i)
+        for i, (name, config) in enumerate(sites)
+    ]
+
+    results = await asyncio.gather(*tasks)
+    return results
+
+
+def print_progress(current: int, total: int, site_name: str):
+    """Print progress indicator."""
+    pct = int(current / total * 100)
+    bar_width = 30
+    filled = int(bar_width * current / total)
+    bar = "█" * filled + "░" * (bar_width - filled)
+    print(f"\r[{bar}] {pct:3d}% ({current}/{total}) {site_name:<30}", end="", flush=True)
+
+
+def generate_report(results: List[SiteCheckResult]) -> dict:
+    """Generate a summary report from check results."""
+    report = {
+        "summary": {
+            "total": len(results),
+            "working": 0,
+            "broken": 0,
+            "disabled": 0,
+            "timeout": 0,
+            "anti_bot": 0,
+            "error": 0,
+            "unknown": 0,
+        },
+        "by_status": defaultdict(list),
+        "issues": [],
+        "recommendations": [],
+    }
+
+    for r in results:
+        report["summary"][r.status] = report["summary"].get(r.status, 0) + 1
+        report["by_status"][r.status].append(r.site_name)
+
+        if r.issues:
+            report["issues"].append({
+                "site": r.site_name,
+                "rank": r.alexa_rank,
+                "issues": r.issues,
+            })
+
+        if r.recommendations:
+            report["recommendations"].append({
+                "site": r.site_name,
+                "rank": r.alexa_rank,
+                "recommendations": r.recommendations,
+            })
+
+    return report
+
+
+def print_report(report: dict, results: List[SiteCheckResult]):
+    """Print a formatted report to console."""
+    summary = report["summary"]
+
+    print(f"\n{'='*60}")
+    print(f"{color('SITE CHECK REPORT', Colors.CYAN)}")
+    print(f"{'='*60}\n")
+
+    print(f"{color('SUMMARY:', Colors.BOLD)}")
+    print(f"  Total sites checked: {summary['total']}")
+    print(f"  {color('Working:', Colors.GREEN)} {summary['working']}")
+    print(f"  {color('Broken:', Colors.RED)} {summary['broken']}")
+    print(f"  {color('Disabled:', Colors.YELLOW)} {summary['disabled']}")
+    print(f"  {color('Timeout:', Colors.YELLOW)} {summary['timeout']}")
+    print(f"  {color('Anti-bot:', Colors.YELLOW)} {summary['anti_bot']}")
+    print(f"  {color('Error:', Colors.RED)} {summary['error']}")
+
+    # Broken sites
+    if report["by_status"]["broken"]:
+        print(f"\n{color('BROKEN SITES:', Colors.RED)}")
+        for site in report["by_status"]["broken"][:20]:
+            r = next(x for x in results if x.site_name == site)
+            print(f"  - {site} (rank {r.alexa_rank}): {', '.join(r.issues)}")
+        if len(report["by_status"]["broken"]) > 20:
+            print(f"  ... and {len(report['by_status']['broken']) - 20} more")
+
+    # Timeout sites
+    if report["by_status"]["timeout"]:
+        print(f"\n{color('TIMEOUT SITES:', Colors.YELLOW)}")
+        for site in report["by_status"]["timeout"][:10]:
+            print(f"  - {site}")
+        if len(report["by_status"]["timeout"]) > 10:
+            print(f"  ... and {len(report['by_status']['timeout']) - 10} more")
+
+    # Anti-bot sites
+    if report["by_status"]["anti_bot"]:
+        print(f"\n{color('ANTI-BOT PROTECTED:', Colors.YELLOW)}")
+        for site in report["by_status"]["anti_bot"][:10]:
+            r = next(x for x in results if x.site_name == site)
+            print(f"  - {site}: {', '.join(r.issues)}")
+        if len(report["by_status"]["anti_bot"]) > 10:
+            print(f"  ... and {len(report['by_status']['anti_bot']) - 10} more")
+
+    # Recommendations
+    if report["recommendations"]:
+        print(f"\n{color('RECOMMENDATIONS:', Colors.CYAN)}")
+        for rec in report["recommendations"][:15]:
+            print(f"  {rec['site']} (rank {rec['rank']}):")
+            for r in rec["recommendations"]:
+                print(f"    -> {r}")
+        if len(report["recommendations"]) > 15:
+            print(f"  ... and {len(report['recommendations']) - 15} more")
+
+
+async def main():
+    parser = argparse.ArgumentParser(
+        description="Mass site checking for Maigret",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--top", "-n", type=int, default=100,
+                        help="Check top N sites by Alexa rank (default: 100)")
+    parser.add_argument("--parallel", "-p", type=int, default=5,
+                        help="Number of parallel requests (default: 5)")
+    parser.add_argument("--timeout", "-t", type=int, default=15,
+                        help="Request timeout in seconds (default: 15)")
+    parser.add_argument("--output", "-o", help="Output JSON report to file")
+    parser.add_argument("--include-disabled", action="store_true",
+                        help="Include disabled sites in results")
+    parser.add_argument("--only-broken", action="store_true",
+                        help="Only show broken sites")
+    parser.add_argument("--json", action="store_true",
+                        help="Output as JSON only")
+
+    args = parser.parse_args()
+
+    # Load sites
+    db_path = Path(__file__).parent.parent / "maigret" / "resources" / "data.json"
+    if not db_path.exists():
+        print(f"Database not found: {db_path}")
+        sys.exit(1)
+
+    sites = load_sites(db_path)
+    top_sites = get_top_sites(sites, args.top)
+
+    if not args.json:
+        print(f"Checking top {len(top_sites)} sites (parallel={args.parallel}, timeout={args.timeout}s)...")
+        print()
+
+    # Run checks
+    progress = print_progress if not args.json else None
+    results = await check_sites_batch(top_sites, args.parallel, args.timeout, progress)
+
+    if not args.json:
+        print()  # Clear progress line
+
+    # Filter results
+    if not args.include_disabled:
+        results = [r for r in results if r.status != "disabled"]
+    if args.only_broken:
+        results = [r for r in results if r.status in ("broken", "error", "timeout")]
+
+    # Generate report
+    report = generate_report(results)
+
+    # Output
+    if args.json:
+        output = {
+            "report": report,
+            "results": [asdict(r) for r in results],
+        }
+        print(json.dumps(output, indent=2))
+    else:
+        print_report(report, results)
+
+    # Save to file
+    if args.output:
+        output = {
+            "report": report,
+            "results": [asdict(r) for r in results],
+        }
+        with open(args.output, "w") as f:
+            json.dump(output, f, indent=2)
+        print(f"\nReport saved to: {args.output}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

utils/cloudshell_install.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+sudo apt-get update && sudo apt-get install -y libcairo2-dev pkg-config
+pip install .

utils/fp_probe_top_sites.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+"""
+Probe likely false-positive sites among the top-N Alexa-ranked entries.
+
+For each of K random *distinct* usernames taken from ``usernameClaimed`` fields in
+the Maigret database, runs a clean ``maigret`` scan (``--top-sites N --json simple|ndjson``).
+Sites that return CLAIMED in *every* run are reported: unrelated random claimed
+handles are unlikely to all exist on the same third-party site, so such sites are
+candidates for broken checks.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def repo_root() -> Path:
+    return Path(__file__).resolve().parent.parent
+
+
+def load_username_claimed_pool(db_path: Path) -> list[str]:
+    with db_path.open(encoding="utf-8") as f:
+        data = json.load(f)
+    sites = data.get("sites") or {}
+    seen: set[str] = set()
+    pool: list[str] = []
+    for _name, site in sites.items():
+        u = (site or {}).get("usernameClaimed")
+        if not u or not isinstance(u, str):
+            continue
+        u = u.strip()
+        if not u or u in seen:
+            continue
+        seen.add(u)
+        pool.append(u)
+    return pool
+
+
+def run_maigret(
+    *,
+    username: str,
+    db_path: Path,
+    out_dir: Path,
+    top_sites: int,
+    json_format: str,
+    quiet: bool,
+) -> Path:
+    """Run maigret subprocess; return path to the written JSON report."""
+    safe = username.replace("/", "_")
+    report_name = f"report_{safe}_{json_format}.json"
+    report_path = out_dir / report_name
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "maigret",
+        username,
+        "--db",
+        str(db_path),
+        "--top-sites",
+        str(top_sites),
+        "--json",
+        json_format,
+        "--folderoutput",
+        str(out_dir),
+        "--no-progressbar",
+        "--no-color",
+        "--no-recursion",
+        "--no-extracting",
+    ]
+    sink = subprocess.DEVNULL if quiet else None
+    proc = subprocess.run(
+        cmd,
+        cwd=str(repo_root()),
+        text=True,
+        stdout=sink,
+        stderr=sink,
+    )
+    if proc.returncode != 0:
+        raise RuntimeError(
+            f"maigret exited with {proc.returncode} for username {username!r}"
+        )
+    if not report_path.is_file():
+        raise FileNotFoundError(f"Expected report missing: {report_path}")
+    return report_path
+
+
+def claimed_sites_from_report(path: Path, json_format: str) -> set[str]:
+    if json_format == "simple":
+        with path.open(encoding="utf-8") as f:
+            data = json.load(f)
+        if not isinstance(data, dict):
+            return set()
+        return set(data.keys())
+    # ndjson: one object per line, each has "sitename"
+    sites: set[str] = set()
+    with path.open(encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            obj = json.loads(line)
+            name = obj.get("sitename")
+            if isinstance(name, str) and name:
+                sites.add(name)
+    return sites
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Pick random distinct usernameClaimed values, run maigret --top-sites N "
+            "with JSON reports, and list sites that claimed all of them (suspicious FP)."
+        )
+    )
+    parser.add_argument(
+        "--db",
+        "-b",
+        type=Path,
+        default=repo_root() / "maigret" / "resources" / "data.json",
+        help="Path to Maigret data.json (a temp copy is used for runs).",
+    )
+    parser.add_argument(
+        "--top-sites",
+        "-n",
+        type=int,
+        default=500,
+        metavar="N",
+        help="Value for maigret --top-sites (default: 500).",
+    )
+    parser.add_argument(
+        "--samples",
+        "-k",
+        type=int,
+        default=5,
+        metavar="K",
+        help="How many distinct random usernames to draw (default: 5).",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=None,
+        help="RNG seed for reproducible username selection.",
+    )
+    parser.add_argument(
+        "--json",
+        dest="json_format",
+        default="simple",
+        choices=["simple", "ndjson"],
+        help="JSON report type passed to maigret -J (default: simple).",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        default=False,
+        help="Print maigret stdout/stderr (default: suppress child output).",
+    )
+    args = parser.parse_args()
+    quiet = not args.verbose
+
+    db_src = args.db.resolve()
+    if not db_src.is_file():
+        print(f"Database not found: {db_src}", file=sys.stderr)
+        return 2
+
+    pool = load_username_claimed_pool(db_src)
+    if len(pool) < args.samples:
+        print(
+            f"Need at least {args.samples} distinct usernameClaimed entries, "
+            f"found {len(pool)}.",
+            file=sys.stderr,
+        )
+        return 2
+
+    rng = random.Random(args.seed)
+    picked = rng.sample(pool, args.samples)
+
+    print(f"Database: {db_src}")
+    print(f"--top-sites {args.top_sites}, {args.samples} random usernameClaimed:")
+    for i, u in enumerate(picked, 1):
+        print(f"  {i}. {u}")
+
+    site_sets: list[set[str]] = []
+    with tempfile.TemporaryDirectory(prefix="maigret_fp_probe_") as tmp:
+        tmp_path = Path(tmp)
+        db_work = tmp_path / "data.json"
+        shutil.copyfile(db_src, db_work)
+
+        for u in picked:
+            print(f"\nRunning maigret for {u!r} ...", flush=True)
+            report = run_maigret(
+                username=u,
+                db_path=db_work,
+                out_dir=tmp_path,
+                top_sites=args.top_sites,
+                json_format=args.json_format,
+                quiet=quiet,
+            )
+            sites = claimed_sites_from_report(report, args.json_format)
+            site_sets.append(sites)
+            print(f"  -> {len(sites)} positive site(s) in JSON", flush=True)
+
+    always = set.intersection(*site_sets) if site_sets else set()
+    print("\n--- Sites with CLAIMED in all runs (candidates for false positives) ---")
+    if not always:
+        print("(none)")
+    else:
+        for name in sorted(always):
+            print(name)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

utils/generate_changelog.py

@@ -0,0 +1,143 @@
+"""Fill the CHANGELOG.md "What's Changed" section for a release.
+
+Reproduces exactly what the GitHub "Generate release notes" button produces
+(``* <PR title> by @<author> in <url>`` bullets, plus New Contributors and the
+Full Changelog footer) and drops it under a ``## [<version>] - <date>`` heading
+at the top of CHANGELOG.md — the same shape every prior release entry uses.
+
+The notes are fetched via the GitHub REST API through the ``gh`` CLI, so it uses
+your existing ``gh auth`` credentials (or GITHUB_TOKEN in CI). No PR data is
+invented locally: the source of truth is GitHub, identical to how the section
+was produced before.
+
+Usage:
+    python utils/generate_changelog.py                 # version from __version__.py, prev tag auto
+    python utils/generate_changelog.py --version 0.6.2 --previous-tag v0.6.1
+    python utils/generate_changelog.py --print         # print notes, don't touch CHANGELOG.md
+"""
+
+import argparse
+import json
+import os.path as path
+import re
+import subprocess
+from datetime import date as date_cls
+from typing import Optional
+
+REPO_ROOT = path.dirname(path.dirname(path.abspath(__file__)))
+VERSION_FILE = path.join(REPO_ROOT, "maigret", "__version__.py")
+CHANGELOG_PATH = path.join(REPO_ROOT, "CHANGELOG.md")
+
+
+def get_current_version() -> str:
+    """Read __version__ from maigret/__version__.py."""
+    with open(VERSION_FILE) as f:
+        for line in f:
+            if line.startswith("__version__"):
+                return line.split("=")[1].strip().strip("'\"")
+    raise RuntimeError(f"__version__ not found in {VERSION_FILE}")
+
+
+def _run(cmd: list) -> str:
+    return subprocess.run(cmd, check=True, capture_output=True, text=True).stdout
+
+
+def get_repo_slug() -> str:
+    """Return 'owner/repo' parsed from the origin remote."""
+    url = _run(["git", "-C", REPO_ROOT, "remote", "get-url", "origin"]).strip()
+    m = re.search(r"github\.com[:/]([^/]+/[^/]+?)(?:\.git)?$", url)
+    if not m:
+        raise RuntimeError(f"Cannot parse GitHub repo from remote url: {url}")
+    return m.group(1)
+
+
+def get_latest_tag() -> Optional[str]:
+    """Latest v* tag by creation date, or None if there are no version tags."""
+    out = _run(["git", "-C", REPO_ROOT, "tag", "--list", "v*", "--sort=-creatordate"])
+    tags = [t for t in out.splitlines() if t.strip()]
+    return tags[0] if tags else None
+
+
+def fetch_release_notes(repo: str, tag_name: str, previous_tag: Optional[str], commitish: str) -> str:
+    """Fetch GitHub auto-generated release notes body for the given tag range.
+
+    Works even when ``tag_name`` does not exist yet by pinning ``target_commitish``.
+    """
+    cmd = [
+        "gh", "api", "--method", "POST",
+        f"repos/{repo}/releases/generate-notes",
+        "-f", f"tag_name={tag_name}",
+        "-f", f"target_commitish={commitish}",
+    ]
+    if previous_tag:
+        cmd += ["-f", f"previous_tag_name={previous_tag}"]
+    body = json.loads(_run(cmd))["body"].strip()
+    return body
+
+
+def build_section(version: str, release_date: str, notes_body: str) -> str:
+    """Assemble one CHANGELOG entry: '## [version] - date' + the notes body."""
+    return f"## [{version}] - {release_date}\n\n{notes_body.strip()}\n"
+
+
+# Matches a version heading like "## [0.6.2] - 2026-07-01" up to (not including)
+# the next version heading or end of file.
+_SECTION_RE_TMPL = r"^## \[{version}\][^\n]*\n.*?(?=^## \[|\Z)"
+
+
+def upsert_section(changelog: str, version: str, section: str) -> str:
+    """Insert ``section`` for ``version`` into ``changelog``.
+
+    If an entry for ``version`` already exists (e.g. an empty placeholder left by
+    the version bump) it is replaced; otherwise the section is inserted directly
+    after the top-level '# Changelog' title. Idempotent.
+    """
+    section = section.rstrip("\n") + "\n"
+    pattern = re.compile(_SECTION_RE_TMPL.format(version=re.escape(version)), re.DOTALL | re.MULTILINE)
+    if pattern.search(changelog):
+        return pattern.sub(lambda _: section + "\n", changelog, count=1)
+
+    # No existing entry: insert after the "# Changelog" header.
+    title = re.compile(r"^# Changelog[^\n]*\n", re.MULTILINE)
+    m = title.search(changelog)
+    if not m:
+        return section + "\n" + changelog
+    idx = m.end()
+    return changelog[:idx] + "\n" + section + "\n" + changelog[idx:].lstrip("\n")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Fill CHANGELOG.md What's Changed from GitHub release notes")
+    parser.add_argument("--version", default=None, help="Release version (default: read from __version__.py)")
+    parser.add_argument("--previous-tag", default=None, help="Previous git tag (default: latest v* tag)")
+    parser.add_argument("--date", default=None, help="Release date YYYY-MM-DD (default: today)")
+    parser.add_argument("--commitish", default="HEAD", help="Target commit/branch for the release (default: HEAD)")
+    parser.add_argument("--repo", default=None, help="owner/repo (default: parsed from origin remote)")
+    parser.add_argument("--print", dest="print_only", action="store_true", help="Print notes only; do not edit CHANGELOG.md")
+    args = parser.parse_args()
+
+    version = args.version or get_current_version()
+    tag_name = f"v{version}"
+    previous_tag = args.previous_tag or get_latest_tag()
+    release_date = args.date or date_cls.today().isoformat()
+    repo = args.repo or get_repo_slug()
+
+    notes = fetch_release_notes(repo, tag_name, previous_tag, args.commitish)
+    section = build_section(version, release_date, notes)
+
+    if args.print_only:
+        print(section)
+        return
+
+    with open(CHANGELOG_PATH, "r", encoding="utf-8") as f:
+        changelog = f.read()
+    updated = upsert_section(changelog, version, section)
+    with open(CHANGELOG_PATH, "w", encoding="utf-8") as f:
+        f.write(updated)
+
+    n_prs = notes.count("\n* ") + (1 if notes.lstrip().startswith("* ") else 0)
+    print(f"Updated {CHANGELOG_PATH}: {version} ({previous_tag or 'start'}..{tag_name}), ~{n_prs} entries")
+
+
+if __name__ == "__main__":
+    main()

utils/generate_db_meta.py

@@ -0,0 +1,123 @@
+"""Generate db_meta.json from data.json for the auto-update system."""
+
+import argparse
+import hashlib
+import json
+import os.path as path
+from datetime import datetime, timezone
+from typing import Optional, Tuple
+
+RESOURCES_DIR = path.join(path.dirname(path.dirname(path.abspath(__file__))), "maigret", "resources")
+DATA_JSON_PATH = path.join(RESOURCES_DIR, "data.json")
+META_JSON_PATH = path.join(RESOURCES_DIR, "db_meta.json")
+DEFAULT_DATA_URL = "https://raw.githubusercontent.com/soxoj/maigret/main/maigret/resources/data.json"
+
+# Backward-compatibility floor: the first maigret version that shipped the db_meta
+# auto-update mechanism (data format "version": 1). min_maigret_version must NOT
+# track the current release — it only rises when a breaking data-format change
+# genuinely requires a newer client. Used as the fallback when no db_meta.json
+# exists yet to read the value from.
+DEFAULT_MIN_VERSION = "0.5.0"
+
+_TIMESTAMP_KEY = "updated_at"
+
+
+def build_meta(data_path: str, min_version: str, data_url: str, now: Optional[datetime] = None) -> dict:
+    """Build a db_meta dict for the given data.json. Does not touch the filesystem."""
+    with open(data_path, "rb") as f:
+        raw = f.read()
+    data = json.loads(raw)
+    ts = (now or datetime.now(timezone.utc)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    return {
+        "version": 1,
+        _TIMESTAMP_KEY: ts,
+        "sites_count": len(data.get("sites", {})),
+        "min_maigret_version": min_version,
+        "data_sha256": hashlib.sha256(raw).hexdigest(),
+        "data_url": data_url,
+    }
+
+
+def meta_payload_equals(a: dict, b: dict) -> bool:
+    """Compare two db_meta dicts ignoring the volatile 'updated_at' field."""
+    a_clean = {k: v for k, v in a.items() if k != _TIMESTAMP_KEY}
+    b_clean = {k: v for k, v in b.items() if k != _TIMESTAMP_KEY}
+    return a_clean == b_clean
+
+
+def _read_meta(meta_path: str) -> Optional[dict]:
+    try:
+        with open(meta_path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (OSError, ValueError):
+        return None
+
+
+def write_meta_if_changed(
+    data_path: str,
+    meta_path: str,
+    min_version: str,
+    data_url: str,
+    now: Optional[datetime] = None,
+) -> Tuple[dict, bool]:
+    """Generate db_meta.json next to data.json. Skip the write entirely if
+    the only thing that would change is `updated_at` — keeps the file (and
+    git/precommit hooks) quiet when the underlying site database hasn't
+    actually moved.
+
+    Returns the meta dict that *would* be written and a bool indicating
+    whether a write happened.
+    """
+    new_meta = build_meta(data_path, min_version, data_url, now=now)
+    existing = _read_meta(meta_path)
+    if existing is not None and meta_payload_equals(existing, new_meta):
+        return existing, False
+
+    with open(meta_path, "w", encoding="utf-8") as f:
+        json.dump(new_meta, f, indent=4, ensure_ascii=False)
+    return new_meta, True
+
+
+def resolve_min_version(cli_min_version: Optional[str], meta_path: str) -> str:
+    """Resolve the min_maigret_version to write.
+
+    Priority: explicit CLI value > value already in db_meta.json > DEFAULT_MIN_VERSION.
+    Crucially this does NOT fall back to the current release version, so routine
+    regenerations (e.g. after a version bump) never silently raise the compatibility
+    floor. It only moves when a maintainer passes --min-version for a real
+    data-format change.
+    """
+    if cli_min_version:
+        return cli_min_version
+    existing = _read_meta(meta_path)
+    if existing and existing.get("min_maigret_version"):
+        return existing["min_maigret_version"]
+    return DEFAULT_MIN_VERSION
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate db_meta.json from data.json")
+    parser.add_argument(
+        "--min-version",
+        default=None,
+        help="Minimum compatible maigret version. Default: preserve the value already "
+        f"in db_meta.json, or {DEFAULT_MIN_VERSION} if none exists. Do NOT let this track "
+        "the release version — only raise it on a breaking data-format change.",
+    )
+    parser.add_argument("--data-url", default=DEFAULT_DATA_URL, help="URL where data.json can be downloaded")
+    args = parser.parse_args()
+
+    min_version = resolve_min_version(args.min_version, META_JSON_PATH)
+    meta, written = write_meta_if_changed(DATA_JSON_PATH, META_JSON_PATH, min_version, args.data_url)
+
+    if written:
+        print(f"Generated {META_JSON_PATH}")
+    else:
+        print(f"Skipped {META_JSON_PATH}: nothing changed except timestamp")
+    print(f"  sites: {meta['sites_count']}")
+    print(f"  sha256: {meta['data_sha256'][:16]}...")
+    print(f"  min_version: {meta['min_maigret_version']}")
+
+
+if __name__ == "__main__":
+    main()

utils/import_sites.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+import json
+import random
+import re
+
+from unittest.mock import Mock
+
+import alive_progress
+import requests
+
+from maigret.maigret import *
+from maigret.result import MaigretCheckStatus
+from maigret.sites import MaigretSite
+
+URL_RE = re.compile(r"https?://(www\.)?")
+TIMEOUT = 200
+
+
+async def maigret_check(site, site_data, username, status, logger):
+    query_notify = Mock()
+    logger.debug(f'Checking {site}...')
+
+    for username, status in [(username, status)]:
+        results = await maigret(
+            username,
+            {site: site_data},
+            logger,
+            query_notify,
+            timeout=TIMEOUT,
+            forced=True,
+            no_progressbar=True,
+        )
+
+        if results[site]['status'].status != status:
+            if results[site]['status'].status == MaigretCheckStatus.UNKNOWN:
+                msg = site_data.absence_strs
+                etype = site_data.check_type
+                context = results[site]['status'].context
+
+                logger.debug(f'Error while searching {username} in {site}, must be claimed. Context: {context}')
+                # if site_data.get('errors'):
+                #     continue
+                return False
+
+            if status == MaigretCheckStatus.CLAIMED:
+                logger.debug(f'Not found {username} in {site}, must be claimed')
+                logger.debug(results[site])
+                pass
+            else:
+                logger.debug(f'Found {username} in {site}, must be available')
+                logger.debug(results[site])
+                pass
+            return False
+
+    return site_data
+
+
+async def check_and_add_maigret_site(site_data, semaphore, logger, ok_usernames, bad_usernames):
+    async with semaphore:
+        sitename = site_data.name
+        positive = False
+        negative = False
+
+        for ok_username in ok_usernames:
+            site_data.username_claimed = ok_username
+            status = MaigretCheckStatus.CLAIMED
+            if await maigret_check(sitename, site_data, ok_username, status, logger):
+                # print(f'{sitename} positive case is okay')
+                positive = True
+                break
+
+        for bad_username in bad_usernames:
+            site_data.username_unclaimed = bad_username
+            status = MaigretCheckStatus.AVAILABLE
+            if await maigret_check(sitename, site_data, bad_username, status, logger):
+                # print(f'{sitename} negative case is okay')
+                negative = True
+                break
+
+        if positive and negative:
+            site_data = site_data.strip_engine_data()
+
+            db.update_site(site_data)
+            print(site_data.json)
+            try:
+                db.save_to_file(args.base_file)
+            except Exception as e:
+                logging.error(e, exc_info=True)
+            print(f'Saved new site {sitename}...')
+            ok_sites.append(site_data)
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter
+                            )
+    parser.add_argument("--base", "-b", metavar="BASE_FILE",
+                        dest="base_file", default="maigret/resources/data.json",
+                        help="JSON file with sites data to update.")
+
+    parser.add_argument("--add-engine", dest="add_engine", help="Additional engine to check")
+
+    parser.add_argument("--only-engine", dest="only_engine", help="Use only this engine from detected to check")
+
+    parser.add_argument('--check', help='only check sites in database', action='store_true')
+
+    parser.add_argument('--random', help='shuffle list of urls', action='store_true', default=False)
+
+    parser.add_argument('--top', help='top count of records in file', type=int, default=10000)
+
+    parser.add_argument('--filter', help='substring to filter input urls', type=str, default='')
+
+    parser.add_argument('--username', help='preferable username to check with', type=str)
+
+    parser.add_argument(
+        "--info",
+        "-vv",
+        action="store_true",
+        dest="info",
+        default=False,
+        help="Display service information.",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        dest="verbose",
+        default=False,
+        help="Display extra information and metrics.",
+    )
+    parser.add_argument(
+        "-d",
+        "--debug",
+        "-vvv",
+        action="store_true",
+        dest="debug",
+        default=False,
+        help="Saving debugging information and sites responses in debug.txt.",
+    )
+
+    parser.add_argument("urls_file",
+                        metavar='URLS_FILE',
+                        action="store",
+                        help="File with base site URLs"
+                        )
+
+    args = parser.parse_args()
+
+    log_level = logging.ERROR
+    if args.debug:
+        log_level = logging.DEBUG
+    elif args.info:
+        log_level = logging.INFO
+    elif args.verbose:
+        log_level = logging.WARNING
+
+    logging.basicConfig(
+        format='[%(filename)s:%(lineno)d] %(levelname)-3s  %(asctime)s %(message)s',
+        datefmt='%H:%M:%S',
+        level=log_level
+    )
+    logger = logging.getLogger('engines-check')
+    logger.setLevel(log_level)
+
+    db = MaigretDatabase()
+    sites_subset = db.load_from_file(args.base_file).sites
+    sites = {site.name: site for site in sites_subset}
+    engines = db.engines
+
+    ok_usernames = ['alex', 'god', 'admin', 'red', 'blue', 'john']
+    if args.username:
+        ok_usernames = [args.username] + ok_usernames
+
+    bad_usernames = ['noonewouldeverusethis7']
+
+    with open(args.urls_file, 'r') as urls_file:
+        urls = urls_file.read().splitlines()
+        if args.random:
+            random.shuffle(urls)
+        urls = urls[:args.top]
+
+    raw_maigret_data = json.dumps({site.name: site.json for site in sites_subset})
+
+    new_sites = []
+    for site in alive_progress.alive_it(urls):
+        site_lowercase = site.lower()
+
+        domain_raw = URL_RE.sub('', site_lowercase).strip().strip('/')
+        domain_raw = domain_raw.split('/')[0]
+
+        if args.filter and args.filter not in domain_raw:
+            logger.debug('Site %s skipped due to filtering by "%s"', domain_raw, args.filter)
+            continue
+
+        if domain_raw in raw_maigret_data:
+            logger.debug(f'Site {domain_raw} already exists in the Maigret database!')
+            continue
+
+        if '"' in domain_raw:
+            logger.debug(f'Invalid site {domain_raw}')
+            continue
+
+        main_page_url = '/'.join(site.split('/', 3)[:3])
+
+        site_data = {
+            'url': site,
+            'urlMain': main_page_url,
+            'name': domain_raw,
+        }
+
+        try:
+            r = requests.get(main_page_url, timeout=5)
+        except:
+            r = None
+            pass
+
+        detected_engines = []
+
+        for e in engines:
+            strs_to_check = e.__dict__.get('presenseStrs')
+            if strs_to_check and r and r.text:
+                all_strs_in_response = True
+                for s in strs_to_check:
+                    if not s in r.text:
+                        all_strs_in_response = False
+                if all_strs_in_response:
+                    engine_name = e.__dict__.get('name')
+                    detected_engines.append(engine_name)
+                    logger.info(f'Detected engine {engine_name} for site {main_page_url}')
+
+        if args.only_engine and args.only_engine in detected_engines:
+            detected_engines = [args.only_engine]
+        elif not detected_engines and args.add_engine:
+            logging.debug('Could not detect any engine, applying default engine %s...', args.add_engine)
+            detected_engines = [args.add_engine]
+
+        def create_site_from_engine(sitename, data, e):
+            site = MaigretSite(sitename, data)
+            site.update_from_engine(db.engines_dict[e])
+            site.engine = e
+            return site
+
+        for engine_name in detected_engines:
+            site = create_site_from_engine(domain_raw, site_data, engine_name)
+            new_sites.append(site)
+            logger.debug(site.json)
+
+            # if engine_name == "phpBB":
+            #     site_data_with_subpath = dict(site_data)
+            #     site_data_with_subpath["urlSubpath"] = "/forum"
+            #     site = create_site_from_engine(domain_raw, site_data_with_subpath, engine_name)
+            #     new_sites.append(site)
+
+        # except Exception as e:
+        #     print(f'Error: {str(e)}')
+        #     pass
+
+    print(f'Found {len(new_sites)}/{len(urls)} new sites')
+
+    if args.check:
+        for s in new_sites:
+            print(s.url_main)
+        sys.exit(0)
+
+    sem = asyncio.Semaphore(20)
+    loop = asyncio.get_event_loop()
+
+    ok_sites = []
+    tasks = []
+    for site in new_sites:
+        check_coro = check_and_add_maigret_site(site, sem, logger, ok_usernames, bad_usernames)
+        future = asyncio.ensure_future(check_coro)
+        tasks.append(future)
+
+    with alive_progress(len(tasks), title='Checking sites') as progress:
+        for f in asyncio.as_completed(tasks):
+            progress()
+        try:
+            loop.run_until_complete(f)
+        except asyncio.exceptions.TimeoutError:
+            pass
+
+    print(f'Found and saved {len(ok_sites)} sites!')

utils/site_check.py

@@ -0,0 +1,808 @@
+#!/usr/bin/env python3
+"""
+Site check utility for Maigret development.
+Quickly test site availability, find valid usernames, and diagnose check issues.
+
+Usage:
+    python utils/site_check.py --site "SiteName" --check-claimed
+    python utils/site_check.py --site "SiteName" --maigret           # Test via Maigret
+    python utils/site_check.py --site "SiteName" --compare-methods   # aiohttp vs Maigret
+    python utils/site_check.py --url "https://example.com/user/{username}" --test "john"
+    python utils/site_check.py --site "SiteName" --find-user
+    python utils/site_check.py --site "SiteName" --diagnose          # Full diagnosis
+"""
+
+import argparse
+import asyncio
+import json
+import logging
+import re
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+# Add parent dir for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+try:
+    import aiohttp
+    from yarl import URL as YarlURL
+except ImportError:
+    print("aiohttp not installed. Run: pip install aiohttp")
+    sys.exit(1)
+
+# Maigret imports (optional, for --maigret mode)
+MAIGRET_AVAILABLE = False
+try:
+    from maigret.sites import MaigretDatabase, MaigretSite
+    from maigret.checking import (
+        SimpleAiohttpChecker,
+        check_site_for_username,
+        process_site_result,
+        make_site_result,
+    )
+    from maigret.notify import QueryNotifyPrint
+    from maigret.result import QueryStatus
+    MAIGRET_AVAILABLE = True
+except ImportError:
+    pass
+
+
+DEFAULT_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+}
+
+COMMON_USERNAMES = ["blue", "test", "admin", "user", "john", "alex", "david", "mike", "chris", "dan"]
+
+
+class Colors:
+    """ANSI color codes for terminal output."""
+    RED = "\033[91m"
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    BLUE = "\033[94m"
+    MAGENTA = "\033[95m"
+    CYAN = "\033[96m"
+    RESET = "\033[0m"
+    BOLD = "\033[1m"
+
+
+def color(text: str, c: str) -> str:
+    """Wrap text with color codes."""
+    return f"{c}{text}{Colors.RESET}"
+
+
+async def check_url_aiohttp(url: str, headers: dict = None, follow_redirects: bool = True,
+                            timeout: int = 15, ssl_verify: bool = False,
+                            method: str = "GET", payload: dict = None) -> dict:
+    """Check a URL using aiohttp and return detailed response info.
+
+    Args:
+        method: HTTP method ("GET" or "POST").
+        payload: JSON payload for POST requests (dict, will be serialized).
+    """
+    headers = headers or DEFAULT_HEADERS.copy()
+    result = {
+        "method": "aiohttp",
+        "url": url,
+        "status": None,
+        "final_url": None,
+        "redirects": [],
+        "content_length": 0,
+        "content": None,
+        "title": None,
+        "error": None,
+        "error_type": None,
+        "markers": {},
+    }
+
+    try:
+        connector = aiohttp.TCPConnector(ssl=ssl_verify)
+        timeout_obj = aiohttp.ClientTimeout(total=timeout)
+
+        async with aiohttp.ClientSession(connector=connector, timeout=timeout_obj) as session:
+            # Use encoded=True if URL contains percent-encoded chars to prevent double-encoding
+            request_url = YarlURL(url, encoded=True) if '%' in url else url
+            request_kwargs = dict(headers=headers, allow_redirects=follow_redirects)
+            if method.upper() == "POST" and payload is not None:
+                request_kwargs["json"] = payload
+
+            request_fn = session.post if method.upper() == "POST" else session.get
+            async with request_fn(request_url, **request_kwargs) as resp:
+                result["status"] = resp.status
+                result["final_url"] = str(resp.url)
+
+                # Get redirect history
+                if resp.history:
+                    result["redirects"] = [str(r.url) for r in resp.history]
+
+                # Read content
+                try:
+                    text = await resp.text()
+                    result["content_length"] = len(text)
+                    result["content"] = text
+
+                    # Extract title
+                    title_match = re.search(r'<title>([^<]*)</title>', text, re.IGNORECASE)
+                    if title_match:
+                        result["title"] = title_match.group(1).strip()[:100]
+
+                    # Check common markers
+                    text_lower = text.lower()
+                    markers = {
+                        "404_text": any(m in text_lower for m in ["not found", "404", "doesn't exist", "does not exist"]),
+                        "profile_markers": any(m in text_lower for m in ["profile", "user", "member", "account"]),
+                        "error_markers": any(m in text_lower for m in ["error", "banned", "suspended", "blocked"]),
+                        "login_required": any(m in text_lower for m in ["log in", "login", "sign in", "signin"]),
+                        "captcha": any(m in text_lower for m in ["captcha", "recaptcha", "challenge", "verify you"]),
+                        "cloudflare": "cloudflare" in text_lower or "cf-ray" in text_lower,
+                        "rate_limit": any(m in text_lower for m in ["rate limit", "too many requests", "429"]),
+                    }
+                    result["markers"] = markers
+
+                    # First 500 chars of body for inspection
+                    result["body_preview"] = text[:500].replace("\n", " ").strip()
+
+                except Exception as e:
+                    result["error"] = f"Content read error: {e}"
+                    result["error_type"] = "content_error"
+
+    except asyncio.TimeoutError:
+        result["error"] = "Timeout"
+        result["error_type"] = "timeout"
+    except aiohttp.ClientError as e:
+        result["error"] = f"Client error: {e}"
+        result["error_type"] = "client_error"
+    except Exception as e:
+        result["error"] = f"Error: {e}"
+        result["error_type"] = "unknown"
+
+    return result
+
+
+async def check_url_maigret(site: 'MaigretSite', username: str, logger=None) -> dict:
+    """Check a URL using Maigret's checking mechanism."""
+    if not MAIGRET_AVAILABLE:
+        return {"error": "Maigret not available", "method": "maigret"}
+
+    if logger is None:
+        logger = logging.getLogger("site_check")
+        logger.setLevel(logging.WARNING)
+
+    result = {
+        "method": "maigret",
+        "url": None,
+        "status": None,
+        "status_str": None,
+        "http_status": None,
+        "final_url": None,
+        "error": None,
+        "error_type": None,
+        "ids_data": None,
+    }
+
+    try:
+        # Create query options
+        options = {
+            "parsing": False,
+            "cookie_jar": None,
+            "timeout": 15,
+        }
+
+        # Create a simple notifier
+        class SilentNotify:
+            def start(self, msg=None): pass
+            def update(self, status, similar=False): pass
+            def finish(self, msg=None, status=None): pass
+
+        notifier = SilentNotify()
+
+        # Run the check
+        site_name, site_result = await check_site_for_username(
+            site, username, options, logger, notifier
+        )
+
+        result["url"] = site_result.get("url_user")
+        result["status"] = site_result.get("status")
+        result["status_str"] = str(site_result.get("status"))
+        result["http_status"] = site_result.get("http_status")
+        result["ids_data"] = site_result.get("ids_data")
+
+        # Check for errors
+        status = site_result.get("status")
+        if status and hasattr(status, 'error') and status.error:
+            result["error"] = f"{status.error.type}: {status.error.desc}"
+            result["error_type"] = str(status.error.type)
+
+    except Exception as e:
+        result["error"] = str(e)
+        result["error_type"] = "exception"
+
+    return result
+
+
+async def find_valid_username(url_template: str, usernames: list = None, headers: dict = None) -> Optional[str]:
+    """Try common usernames to find one that works."""
+    usernames = usernames or COMMON_USERNAMES
+    headers = headers or DEFAULT_HEADERS.copy()
+
+    print(f"Testing {len(usernames)} usernames on {url_template}...")
+
+    for username in usernames:
+        url = url_template.replace("{username}", username)
+        result = await check_url_aiohttp(url, headers)
+
+        status = result["status"]
+        markers = result.get("markers", {})
+
+        # Good signs: 200 status, profile markers, no 404 text
+        if status == 200 and not markers.get("404_text") and markers.get("profile_markers"):
+            print(f"  {color('[+]', Colors.GREEN)} {username}: status={status}, has profile markers")
+            return username
+        elif status == 200 and not markers.get("404_text"):
+            print(f"  {color('[?]', Colors.YELLOW)} {username}: status={status}, might work")
+        else:
+            print(f"  {color('[-]', Colors.RED)} {username}: status={status}")
+
+    return None
+
+
+async def compare_users_aiohttp(url_template: str, claimed: str, unclaimed: str = "noonewouldeverusethis7",
+                                headers: dict = None) -> Tuple[dict, dict]:
+    """Compare responses for claimed vs unclaimed usernames using aiohttp."""
+    headers = headers or DEFAULT_HEADERS.copy()
+
+    print(f"\n{'='*60}")
+    print(f"Comparing: {color(claimed, Colors.GREEN)} vs {color(unclaimed, Colors.RED)}")
+    print(f"URL template: {url_template}")
+    print(f"Method: aiohttp")
+    print(f"{'='*60}\n")
+
+    url_claimed = url_template.replace("{username}", claimed)
+    url_unclaimed = url_template.replace("{username}", unclaimed)
+
+    result_claimed, result_unclaimed = await asyncio.gather(
+        check_url_aiohttp(url_claimed, headers),
+        check_url_aiohttp(url_unclaimed, headers)
+    )
+
+    def print_result(name, r, c):
+        print(f"--- {color(name, c)} ---")
+        print(f"  URL: {r['url']}")
+        print(f"  Status: {color(str(r['status']), Colors.GREEN if r['status'] == 200 else Colors.RED)}")
+        if r["redirects"]:
+            print(f"  Redirects: {' -> '.join(r['redirects'])} -> {r['final_url']}")
+        print(f"  Final URL: {r['final_url']}")
+        print(f"  Content length: {r['content_length']}")
+        print(f"  Title: {r['title']}")
+        if r["error"]:
+            print(f"  Error: {color(r['error'], Colors.RED)}")
+        print(f"  Markers: {r['markers']}")
+        print()
+
+    print_result(f"CLAIMED ({claimed})", result_claimed, Colors.GREEN)
+    print_result(f"UNCLAIMED ({unclaimed})", result_unclaimed, Colors.RED)
+
+    # Analysis
+    print(f"--- {color('ANALYSIS', Colors.CYAN)} ---")
+    recommendations = []
+
+    if result_claimed["status"] != result_unclaimed["status"]:
+        print(f"  [!] Status codes differ: {result_claimed['status']} vs {result_unclaimed['status']}")
+        recommendations.append(("status_code", f"Status codes: {result_claimed['status']} vs {result_unclaimed['status']}"))
+
+    if result_claimed["final_url"] != result_unclaimed["final_url"]:
+        print(f"  [!] Final URLs differ")
+        recommendations.append(("response_url", "Final URLs differ"))
+
+    if result_claimed["content_length"] != result_unclaimed["content_length"]:
+        diff = abs(result_claimed["content_length"] - result_unclaimed["content_length"])
+        print(f"  [!] Content length differs by {diff} bytes")
+        recommendations.append(("message", f"Content differs by {diff} bytes"))
+
+    if result_claimed["title"] != result_unclaimed["title"]:
+        print(f"  [!] Titles differ:")
+        print(f"      Claimed: {result_claimed['title']}")
+        print(f"      Unclaimed: {result_unclaimed['title']}")
+        recommendations.append(("message", f"Titles differ: '{result_claimed['title']}' vs '{result_unclaimed['title']}'"))
+
+    # Check for problems
+    if result_claimed.get("markers", {}).get("captcha"):
+        print(f"  {color('[WARN]', Colors.YELLOW)} Captcha detected on claimed page")
+    if result_claimed.get("markers", {}).get("cloudflare"):
+        print(f"  {color('[WARN]', Colors.YELLOW)} Cloudflare protection detected")
+    if result_claimed.get("markers", {}).get("login_required"):
+        print(f"  {color('[WARN]', Colors.YELLOW)} Login may be required")
+
+    if recommendations:
+        print(f"\n  {color('Recommended checkType:', Colors.BOLD)} {recommendations[0][0]}")
+    else:
+        print(f"  {color('[!]', Colors.RED)} No clear difference found - site may need special handling")
+
+    return result_claimed, result_unclaimed
+
+
+async def compare_methods(site: 'MaigretSite', claimed: str, unclaimed: str) -> dict:
+    """Compare aiohttp vs Maigret results for the same site."""
+    if not MAIGRET_AVAILABLE:
+        print(color("Maigret not available for comparison", Colors.RED))
+        return {}
+
+    print(f"\n{'='*60}")
+    print(f"{color('METHOD COMPARISON', Colors.CYAN)}: aiohttp vs Maigret")
+    print(f"Site: {site.name}")
+    print(f"Claimed: {claimed}, Unclaimed: {unclaimed}")
+    print(f"{'='*60}\n")
+
+    # Build URL template
+    url_template = site.url
+    url_template = url_template.replace("{urlMain}", site.url_main or "")
+    url_template = url_template.replace("{urlSubpath}", getattr(site, 'url_subpath', '') or "")
+
+    headers = DEFAULT_HEADERS.copy()
+    if hasattr(site, 'headers') and site.headers:
+        headers.update(site.headers)
+
+    # Run all checks in parallel
+    url_claimed = url_template.replace("{username}", claimed)
+    url_unclaimed = url_template.replace("{username}", unclaimed)
+
+    aiohttp_claimed, aiohttp_unclaimed, maigret_claimed, maigret_unclaimed = await asyncio.gather(
+        check_url_aiohttp(url_claimed, headers),
+        check_url_aiohttp(url_unclaimed, headers),
+        check_url_maigret(site, claimed),
+        check_url_maigret(site, unclaimed),
+    )
+
+    def status_icon(status):
+        if status == 200:
+            return color("200", Colors.GREEN)
+        elif status == 404:
+            return color("404", Colors.YELLOW)
+        elif status and status >= 400:
+            return color(str(status), Colors.RED)
+        return str(status)
+
+    def maigret_status_icon(status_str):
+        if "Claimed" in str(status_str):
+            return color("Claimed", Colors.GREEN)
+        elif "Available" in str(status_str):
+            return color("Available", Colors.YELLOW)
+        else:
+            return color(str(status_str), Colors.RED)
+
+    print(f"{'Method':<12} {'Username':<25} {'HTTP Status':<12} {'Result':<20}")
+    print("-" * 70)
+    print(f"{'aiohttp':<12} {claimed:<25} {status_icon(aiohttp_claimed['status']):<20} {'OK' if not aiohttp_claimed['error'] else aiohttp_claimed['error'][:20]}")
+    print(f"{'aiohttp':<12} {unclaimed:<25} {status_icon(aiohttp_unclaimed['status']):<20} {'OK' if not aiohttp_unclaimed['error'] else aiohttp_unclaimed['error'][:20]}")
+    print(f"{'Maigret':<12} {claimed:<25} {status_icon(maigret_claimed.get('http_status')):<20} {maigret_status_icon(maigret_claimed.get('status_str'))}")
+    print(f"{'Maigret':<12} {unclaimed:<25} {status_icon(maigret_unclaimed.get('http_status')):<20} {maigret_status_icon(maigret_unclaimed.get('status_str'))}")
+
+    # Check for discrepancies
+    print(f"\n--- {color('DISCREPANCY ANALYSIS', Colors.CYAN)} ---")
+    issues = []
+
+    if aiohttp_claimed['status'] != maigret_claimed.get('http_status'):
+        issues.append(f"HTTP status mismatch for claimed: aiohttp={aiohttp_claimed['status']}, Maigret={maigret_claimed.get('http_status')}")
+
+    if aiohttp_unclaimed['status'] != maigret_unclaimed.get('http_status'):
+        issues.append(f"HTTP status mismatch for unclaimed: aiohttp={aiohttp_unclaimed['status']}, Maigret={maigret_unclaimed.get('http_status')}")
+
+    # Check Maigret detection correctness
+    claimed_detected = "Claimed" in str(maigret_claimed.get('status_str', ''))
+    unclaimed_detected = "Available" in str(maigret_unclaimed.get('status_str', ''))
+
+    if not claimed_detected:
+        issues.append(f"Maigret did NOT detect claimed user '{claimed}' as Claimed")
+    if not unclaimed_detected:
+        issues.append(f"Maigret did NOT detect unclaimed user '{unclaimed}' as Available")
+
+    if issues:
+        for issue in issues:
+            print(f"  {color('[!]', Colors.RED)} {issue}")
+    else:
+        print(f"  {color('[OK]', Colors.GREEN)} Both methods agree on results")
+
+    return {
+        "aiohttp_claimed": aiohttp_claimed,
+        "aiohttp_unclaimed": aiohttp_unclaimed,
+        "maigret_claimed": maigret_claimed,
+        "maigret_unclaimed": maigret_unclaimed,
+        "issues": issues,
+    }
+
+
+async def diagnose_site(site_config: dict, site_name: str) -> dict:
+    """Full diagnosis of a site configuration."""
+    print(f"\n{'='*60}")
+    print(f"{color('FULL SITE DIAGNOSIS', Colors.CYAN)}: {site_name}")
+    print(f"{'='*60}\n")
+
+    diagnosis = {
+        "site_name": site_name,
+        "issues": [],
+        "warnings": [],
+        "recommendations": [],
+        "working": False,
+    }
+
+    # 1. Config analysis
+    print(f"--- {color('1. CONFIGURATION', Colors.BOLD)} ---")
+    check_type = site_config.get("checkType", "status_code")
+    url = site_config.get("url", "")
+    url_main = site_config.get("urlMain", "")
+    claimed = site_config.get("usernameClaimed")
+    unclaimed = site_config.get("usernameUnclaimed", "noonewouldeverusethis7")
+    disabled = site_config.get("disabled", False)
+
+    print(f"  checkType: {check_type}")
+    print(f"  URL: {url}")
+    print(f"  urlMain: {url_main}")
+    print(f"  usernameClaimed: {claimed}")
+    print(f"  disabled: {disabled}")
+
+    if disabled:
+        diagnosis["issues"].append("Site is disabled")
+        print(f"  {color('[!]', Colors.YELLOW)} Site is disabled")
+
+    if not claimed:
+        diagnosis["issues"].append("No usernameClaimed defined")
+        print(f"  {color('[!]', Colors.RED)} No usernameClaimed defined")
+        return diagnosis
+
+    # Build full URL (display URL)
+    url_template = url.replace("{urlMain}", url_main).replace("{urlSubpath}", site_config.get("urlSubpath", ""))
+
+    # Build probe URL (what Maigret actually requests)
+    url_probe = site_config.get("urlProbe", "")
+    if url_probe:
+        probe_template = url_probe.replace("{urlMain}", url_main).replace("{urlSubpath}", site_config.get("urlSubpath", ""))
+    else:
+        probe_template = url_template
+
+    # Detect request method and payload
+    request_method = site_config.get("requestMethod", "GET").upper()
+    request_payload_template = site_config.get("requestPayload")
+
+    headers = DEFAULT_HEADERS.copy()
+    # For API probes (urlProbe, POST), use neutral Accept header instead of text/html
+    # which can cause servers to return HTML instead of JSON
+    if url_probe or request_method == "POST":
+        headers["Accept"] = "*/*"
+    if site_config.get("headers"):
+        headers.update(site_config["headers"])
+
+    if url_probe:
+        print(f"  urlProbe: {url_probe}")
+    if request_method != "GET":
+        print(f"  requestMethod: {request_method}")
+    if request_payload_template:
+        print(f"  requestPayload: {request_payload_template}")
+
+    # 2. Connectivity test
+    print(f"\n--- {color('2. CONNECTIVITY TEST', Colors.BOLD)} ---")
+    probe_claimed = probe_template.replace("{username}", claimed)
+    probe_unclaimed = probe_template.replace("{username}", unclaimed)
+
+    # Build payloads with username substituted
+    payload_claimed = None
+    payload_unclaimed = None
+    if request_payload_template and request_method == "POST":
+        payload_claimed = json.loads(
+            json.dumps(request_payload_template).replace("{username}", claimed)
+        )
+        payload_unclaimed = json.loads(
+            json.dumps(request_payload_template).replace("{username}", unclaimed)
+        )
+
+    result_claimed, result_unclaimed = await asyncio.gather(
+        check_url_aiohttp(probe_claimed, headers, method=request_method, payload=payload_claimed),
+        check_url_aiohttp(probe_unclaimed, headers, method=request_method, payload=payload_unclaimed)
+    )
+
+    print(f"  Claimed ({claimed}): status={result_claimed['status']}, error={result_claimed['error']}")
+    print(f"  Unclaimed ({unclaimed}): status={result_unclaimed['status']}, error={result_unclaimed['error']}")
+
+    # Check for common problems
+    if result_claimed["error_type"] == "timeout":
+        diagnosis["issues"].append("Timeout on claimed username")
+    if result_unclaimed["error_type"] == "timeout":
+        diagnosis["issues"].append("Timeout on unclaimed username")
+
+    if result_claimed.get("markers", {}).get("cloudflare"):
+        diagnosis["warnings"].append("Cloudflare protection detected")
+    if result_claimed.get("markers", {}).get("captcha"):
+        diagnosis["warnings"].append("Captcha detected")
+    if result_claimed["status"] == 403:
+        diagnosis["issues"].append("403 Forbidden - possible anti-bot protection")
+    if result_claimed["status"] == 429:
+        diagnosis["issues"].append("429 Rate Limited")
+
+    # 3. Check type validation
+    print(f"\n--- {color('3. CHECK TYPE VALIDATION', Colors.BOLD)} ---")
+
+    if check_type == "status_code":
+        if result_claimed["status"] == result_unclaimed["status"]:
+            diagnosis["issues"].append(f"status_code check but same status ({result_claimed['status']}) for both")
+            print(f"  {color('[FAIL]', Colors.RED)} Same status code for claimed and unclaimed: {result_claimed['status']}")
+        else:
+            print(f"  {color('[OK]', Colors.GREEN)} Status codes differ: {result_claimed['status']} vs {result_unclaimed['status']}")
+            diagnosis["working"] = True
+
+    elif check_type == "response_url":
+        if result_claimed["final_url"] == result_unclaimed["final_url"]:
+            diagnosis["issues"].append("response_url check but same final URL for both")
+            print(f"  {color('[FAIL]', Colors.RED)} Same final URL for both")
+        else:
+            print(f"  {color('[OK]', Colors.GREEN)} Final URLs differ")
+            diagnosis["working"] = True
+
+    elif check_type == "message":
+        presense_strs = site_config.get("presenseStrs", [])
+        absence_strs = site_config.get("absenceStrs", [])
+
+        print(f"  presenseStrs: {presense_strs}")
+        print(f"  absenceStrs: {absence_strs}")
+
+        claimed_content = result_claimed.get("content", "") or ""
+        unclaimed_content = result_unclaimed.get("content", "") or ""
+
+        # Check presenseStrs
+        presense_found_claimed = any(s in claimed_content for s in presense_strs) if presense_strs else True
+        presense_found_unclaimed = any(s in unclaimed_content for s in presense_strs) if presense_strs else True
+
+        # Check absenceStrs
+        absence_found_claimed = any(s in claimed_content for s in absence_strs) if absence_strs else False
+        absence_found_unclaimed = any(s in unclaimed_content for s in absence_strs) if absence_strs else False
+
+        print(f"  Claimed - presenseStrs found: {presense_found_claimed}, absenceStrs found: {absence_found_claimed}")
+        print(f"  Unclaimed - presenseStrs found: {presense_found_unclaimed}, absenceStrs found: {absence_found_unclaimed}")
+
+        if presense_strs and not presense_found_claimed:
+            diagnosis["issues"].append(f"presenseStrs {presense_strs} not found in claimed page")
+            print(f"  {color('[FAIL]', Colors.RED)} presenseStrs not found in claimed page")
+        if absence_strs and absence_found_claimed:
+            diagnosis["issues"].append(f"absenceStrs {absence_strs} found in claimed page (should not be)")
+            print(f"  {color('[FAIL]', Colors.RED)} absenceStrs found in claimed page")
+        if absence_strs and not absence_found_unclaimed:
+            diagnosis["warnings"].append(f"absenceStrs not found in unclaimed page")
+            print(f"  {color('[WARN]', Colors.YELLOW)} absenceStrs not found in unclaimed page")
+
+        # Check works if: claimed is detected as present AND unclaimed is detected as absent.
+        # Presence detection: presenseStrs found (or empty = always true).
+        # Absence detection: absenceStrs found in unclaimed (or empty = never, rely on presenseStrs only).
+        # With only presenseStrs: works if found in claimed but NOT in unclaimed.
+        # With only absenceStrs: works if found in unclaimed but NOT in claimed.
+        # With both: standard combination.
+        claimed_is_present = presense_found_claimed and not absence_found_claimed
+        unclaimed_is_absent = (
+            (absence_strs and absence_found_unclaimed) or
+            (presense_strs and not presense_found_unclaimed)
+        )
+        if claimed_is_present and unclaimed_is_absent:
+            print(f"  {color('[OK]', Colors.GREEN)} Message check should work correctly")
+            diagnosis["working"] = True
+
+    # 4. Recommendations
+    print(f"\n--- {color('4. RECOMMENDATIONS', Colors.BOLD)} ---")
+
+    if not diagnosis["working"]:
+        # Suggest alternatives
+        if result_claimed["status"] != result_unclaimed["status"]:
+            diagnosis["recommendations"].append(f"Switch to checkType: status_code (status {result_claimed['status']} vs {result_unclaimed['status']})")
+        if result_claimed["final_url"] != result_unclaimed["final_url"]:
+            diagnosis["recommendations"].append("Switch to checkType: response_url")
+        if result_claimed["title"] != result_unclaimed["title"]:
+            diagnosis["recommendations"].append(f"Use title as marker: presenseStrs=['{result_claimed['title']}'] or absenceStrs=['{result_unclaimed['title']}']")
+
+    if diagnosis["recommendations"]:
+        for rec in diagnosis["recommendations"]:
+            print(f"  -> {rec}")
+    elif diagnosis["working"]:
+        print(f"  {color('Site appears to be working correctly', Colors.GREEN)}")
+    else:
+        print(f"  {color('No clear fix found - site may need special handling or should be disabled', Colors.RED)}")
+
+    # Summary
+    print(f"\n--- {color('SUMMARY', Colors.BOLD)} ---")
+    if diagnosis["issues"]:
+        print(f"  Issues: {len(diagnosis['issues'])}")
+        for issue in diagnosis["issues"]:
+            print(f"    - {issue}")
+    if diagnosis["warnings"]:
+        print(f"  Warnings: {len(diagnosis['warnings'])}")
+        for warn in diagnosis["warnings"]:
+            print(f"    - {warn}")
+    print(f"  Working: {color('YES', Colors.GREEN) if diagnosis['working'] else color('NO', Colors.RED)}")
+
+    return diagnosis
+
+
+def load_site_from_db(site_name: str) -> Tuple[Optional[dict], Optional['MaigretSite']]:
+    """Load site config from data.json. Returns (config_dict, MaigretSite or None)."""
+    db_path = Path(__file__).parent.parent / "maigret" / "resources" / "data.json"
+
+    with open(db_path) as f:
+        data = json.load(f)
+
+    config = None
+    if site_name in data["sites"]:
+        config = data["sites"][site_name]
+    else:
+        # Try case-insensitive search
+        for name, cfg in data["sites"].items():
+            if name.lower() == site_name.lower():
+                config = cfg
+                site_name = name
+                break
+
+    if not config:
+        return None, None
+
+    # Also load MaigretSite if available
+    maigret_site = None
+    if MAIGRET_AVAILABLE:
+        try:
+            db = MaigretDatabase().load_from_path(db_path)
+            maigret_site = db.sites_dict.get(site_name)
+        except Exception:
+            pass
+
+    return config, maigret_site
+
+
+async def main():
+    parser = argparse.ArgumentParser(
+        description="Site check utility for Maigret development",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  %(prog)s --site "VK" --check-claimed          # Test site with aiohttp
+  %(prog)s --site "VK" --maigret                # Test site with Maigret
+  %(prog)s --site "VK" --compare-methods        # Compare aiohttp vs Maigret
+  %(prog)s --site "VK" --diagnose               # Full diagnosis
+  %(prog)s --url "https://vk.com/{username}" --compare blue nobody123
+  %(prog)s --site "VK" --find-user              # Find a valid username
+        """
+    )
+    parser.add_argument("--site", "-s", help="Site name from data.json")
+    parser.add_argument("--url", "-u", help="URL template with {username}")
+    parser.add_argument("--test", "-t", help="Username to test")
+    parser.add_argument("--compare", "-c", nargs=2, metavar=("CLAIMED", "UNCLAIMED"),
+                        help="Compare two usernames")
+    parser.add_argument("--find-user", "-f", action="store_true",
+                        help="Find a valid username")
+    parser.add_argument("--check-claimed", action="store_true",
+                        help="Check if claimed username still works (aiohttp)")
+    parser.add_argument("--maigret", "-m", action="store_true",
+                        help="Test using Maigret's checker instead of aiohttp")
+    parser.add_argument("--compare-methods", action="store_true",
+                        help="Compare aiohttp vs Maigret results")
+    parser.add_argument("--diagnose", "-d", action="store_true",
+                        help="Full diagnosis of site configuration")
+    parser.add_argument("--headers", help="Custom headers as JSON")
+    parser.add_argument("--timeout", type=int, default=15, help="Request timeout in seconds")
+    parser.add_argument("--json", action="store_true", help="Output results as JSON")
+
+    args = parser.parse_args()
+
+    url_template = None
+    claimed = None
+    unclaimed = "noonewouldeverusethis7"
+    headers = DEFAULT_HEADERS.copy()
+    site_config = None
+    maigret_site = None
+
+    # Load from site name
+    if args.site:
+        site_config, maigret_site = load_site_from_db(args.site)
+        if not site_config:
+            print(f"Site '{args.site}' not found in database")
+            sys.exit(1)
+
+        url_template = site_config.get("url", "")
+        url_main = site_config.get("urlMain", "")
+        url_subpath = site_config.get("urlSubpath", "")
+        url_template = url_template.replace("{urlMain}", url_main).replace("{urlSubpath}", url_subpath)
+
+        claimed = site_config.get("usernameClaimed")
+        unclaimed = site_config.get("usernameUnclaimed", unclaimed)
+
+        if site_config.get("headers"):
+            headers.update(site_config["headers"])
+
+        if not args.json:
+            print(f"Loaded site: {args.site}")
+            print(f"  URL: {url_template}")
+            print(f"  Claimed: {claimed}")
+            print(f"  CheckType: {site_config.get('checkType', 'unknown')}")
+            print(f"  Disabled: {site_config.get('disabled', False)}")
+
+    # Override with explicit URL
+    if args.url:
+        url_template = args.url
+
+    # Custom headers
+    if args.headers:
+        headers.update(json.loads(args.headers))
+
+    # Actions
+    if args.diagnose:
+        if not site_config:
+            print("--diagnose requires --site")
+            sys.exit(1)
+        result = await diagnose_site(site_config, args.site)
+        if args.json:
+            print(json.dumps(result, indent=2, default=str))
+
+    elif args.compare_methods:
+        if not maigret_site:
+            if not MAIGRET_AVAILABLE:
+                print("Maigret imports not available")
+            else:
+                print("Could not load MaigretSite object")
+            sys.exit(1)
+        result = await compare_methods(maigret_site, claimed, unclaimed)
+        if args.json:
+            print(json.dumps(result, indent=2, default=str))
+
+    elif args.maigret:
+        if not maigret_site:
+            if not MAIGRET_AVAILABLE:
+                print("Maigret imports not available")
+            else:
+                print("Could not load MaigretSite object")
+            sys.exit(1)
+
+        print(f"\n--- Testing with Maigret ---")
+        for username in [claimed, unclaimed]:
+            result = await check_url_maigret(maigret_site, username)
+            print(f"  {username}: status={result.get('status_str')}, http={result.get('http_status')}, error={result.get('error')}")
+
+    elif args.find_user:
+        if not url_template:
+            print("--find-user requires --site or --url")
+            sys.exit(1)
+        result = await find_valid_username(url_template, headers=headers)
+        if result:
+            print(f"\n{color('Found valid username:', Colors.GREEN)} {result}")
+        else:
+            print(f"\n{color('No valid username found', Colors.RED)}")
+
+    elif args.compare:
+        if not url_template:
+            print("--compare requires --site or --url")
+            sys.exit(1)
+        result = await compare_users_aiohttp(url_template, args.compare[0], args.compare[1], headers)
+        if args.json:
+            # Remove content field for JSON output (too large)
+            for r in result:
+                if isinstance(r, dict) and "content" in r:
+                    del r["content"]
+            print(json.dumps(result, indent=2, default=str))
+
+    elif args.check_claimed and claimed:
+        result = await compare_users_aiohttp(url_template, claimed, unclaimed, headers)
+
+    elif args.test:
+        if not url_template:
+            print("--test requires --site or --url")
+            sys.exit(1)
+        url = url_template.replace("{username}", args.test)
+        result = await check_url_aiohttp(url, headers, timeout=args.timeout)
+        if "content" in result:
+            del result["content"]  # Too large for display
+        print(json.dumps(result, indent=2, default=str))
+
+    else:
+        # Default: check claimed username if available
+        if url_template and claimed:
+            await compare_users_aiohttp(url_template, claimed, unclaimed, headers)
+        else:
+            parser.print_help()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

utils/sites_diff.py

@@ -0,0 +1,36 @@
+import sys
+import difflib
+import requests
+
+
+a = requests.get(sys.argv[1]).text
+b = requests.get(sys.argv[2]).text
+
+
+tokens_a = set(a.split('"'))
+tokens_b = set(b.split('"'))
+
+a_minus_b = tokens_a.difference(tokens_b)
+b_minus_a = tokens_b.difference(tokens_a)
+
+print(a_minus_b)
+print(b_minus_a)
+
+print(len(a_minus_b))
+print(len(b_minus_a))
+
+desired_strings = ["username", "not found", "пользователь", "profile", "lastname", "firstname", "biography",
+"birthday", "репутация", "информация", "e-mail"]
+
+
+def get_match_ratio(x):
+    return round(max([
+    	difflib.SequenceMatcher(a=x.lower(), b=y).ratio()
+    	for y in desired_strings
+    ]), 2)
+
+
+RATIO = 0.6
+
+print(sorted(a_minus_b, key=get_match_ratio, reverse=True)[:10])
+print(sorted(b_minus_a, key=get_match_ratio, reverse=True)[:10])

utils/update_site_data.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+"""Maigret: Supported Site Listing with Alexa ranking and country tags
+This module generates the listing of supported sites in file `SITES.md`
+and pretty prints file with sites data.
+"""
+import io
+import os
+import re
+import sys
+import socket
+import requests
+import logging
+import threading
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+
+# Make `from utils.X import Y` work when invoked as `python3 ./utils/update_site_data.py`
+# (direct script execution puts utils/ on sys.path, not the repo root).
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from maigret.maigret import MaigretDatabase
+from utils.generate_db_meta import write_meta_if_changed
+
+SITES_MD_DATE_RE = re.compile(r'\nThe list was updated at \(\d{4}-\d{2}-\d{2}\)\n')
+SITES_MD_DATE_PLACEHOLDER = '\nThe list was updated at (DATE)\n'
+
+
+def sites_md_payload_equals(a: str, b: str) -> bool:
+    """Compare two sites.md bodies ignoring the volatile 'updated at' date."""
+    return SITES_MD_DATE_RE.sub(SITES_MD_DATE_PLACEHOLDER, a) == SITES_MD_DATE_RE.sub(SITES_MD_DATE_PLACEHOLDER, b)
+
+
+def write_sites_md_if_changed(content: str, path: str) -> bool:
+    """Write `content` to `path` only if it differs from the existing file
+    by something other than the 'updated at' date. Returns True if a write
+    happened. Keeps the precommit hook from rewriting the file when the
+    site database itself hasn't moved.
+    """
+    if os.path.exists(path):
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                existing = f.read()
+        except OSError:
+            existing = None
+        if existing is not None and sites_md_payload_equals(existing, content):
+            return False
+
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(content)
+    return True
+
+RANKS = {str(i):str(i) for i in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 50, 100, 500]}
+RANKS.update({
+    '1000': '1K',
+    '5000': '5K',
+    '10000': '10K',
+    '100000': '100K',
+    '10000000': '10M',
+    '50000000': '50M',
+    '100000000': '100M',
+})
+
+
+
+import csv
+import io
+from urllib.parse import urlparse
+
+def fetch_majestic_million():
+    print("Fetching Majestic Million CSV (this may take a few seconds)...")
+    ranks = {}
+    url = "https://downloads.majestic.com/majestic_million.csv"
+    try:
+        response = requests.get(url, stream=True)
+        response.raise_for_status()
+        
+        csv_file = io.StringIO(response.text)
+        reader = csv.reader(csv_file)
+        next(reader) # skip headers
+        
+        for row in reader:
+            if not row or len(row) < 3:
+                continue
+            rank = int(row[0])
+            domain = row[2].lower()
+            ranks[domain] = rank
+    except Exception as e:
+        logging.error(f"Error fetching Majestic Million: {e}")
+        
+    print(f"Loaded {len(ranks)} domains from Majestic Million.")
+    return ranks
+
+def get_base_domain(url):
+    try:
+        netloc = urlparse(url).netloc
+        if netloc.startswith('www.'):
+            netloc = netloc[4:]
+        return netloc.lower()
+    except Exception:
+        return ""
+
+
+def check_dns(domain, timeout=5):
+    """Check if a domain resolves via DNS. Returns True if it resolves."""
+    try:
+        socket.setdefaulttimeout(timeout)
+        socket.getaddrinfo(domain, None)
+        return True
+    except (socket.gaierror, socket.timeout, OSError):
+        return False
+
+
+def check_sites_dns(sites):
+    """Check DNS resolution for all sites. Returns a set of site names that failed."""
+    SKIP_TLDS = ('.onion', '.i2p')
+    domains = {}
+    for site in sites:
+        domain = get_base_domain(site.url_main)
+        if domain and not any(domain.endswith(tld) for tld in SKIP_TLDS):
+            domains.setdefault(domain, []).append(site)
+
+    failed_sites = set()
+    results = {}
+
+    def resolve(domain):
+        results[domain] = check_dns(domain)
+
+    threads = []
+    for domain in domains:
+        t = threading.Thread(target=resolve, args=(domain,))
+        threads.append(t)
+        t.start()
+
+    for t in threads:
+        t.join()
+
+    for domain, resolved in results.items():
+        if not resolved:
+            for site in domains[domain]:
+                failed_sites.add(site.name)
+            logging.warning(f"DNS resolution failed for {domain}")
+
+    return failed_sites
+
+
+def get_step_rank(rank):
+    def get_readable_rank(r):
+        return RANKS[str(r)]
+
+    valid_step_ranks = sorted(map(int, RANKS.keys()))
+    if rank == 0 or rank == sys.maxsize:
+        return get_readable_rank(valid_step_ranks[-1])
+    else:
+        return get_readable_rank(list(filter(lambda x: x >= rank, valid_step_ranks))[0])
+
+
+def main():
+    parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter
+                            )
+    parser.add_argument("--base","-b", metavar="BASE_FILE",
+                        dest="base_file", default="maigret/resources/data.json",
+                        help="JSON file with sites data to update.")
+
+    parser.add_argument('--with-rank', help='update with use of local data only', action='store_true')
+    parser.add_argument('--empty-only', help='update only sites without rating', action='store_true')
+    parser.add_argument('--exclude-engine', help='do not update score with certain engine',
+                        action="append", dest="exclude_engine_list", default=[])
+    parser.add_argument('--dns-check', help='disable sites whose domains do not resolve via DNS',
+                        action='store_true')
+
+    pool = list()
+
+    args = parser.parse_args()
+
+    db = MaigretDatabase()
+    sites_subset = db.load_from_file(args.base_file).sites
+
+    print(f"\nUpdating supported sites list (don't worry, it's needed)...")
+
+    site_file = io.StringIO()
+    site_file.write(f"""
+## List of supported sites (search methods): total {len(sites_subset)}\n
+Rank data fetched from Majestic Million by domains.
+
+""")
+
+    if args.dns_check:
+        print("Checking DNS resolution for all site domains...")
+        failed = check_sites_dns(sites_subset)
+        disabled_count = 0
+        re_enabled_count = 0
+        for site in sites_subset:
+            if site.name in failed:
+                if not site.disabled:
+                    site.disabled = True
+                    disabled_count += 1
+                    print(f"  Disabled {site.name}: DNS does not resolve ({get_base_domain(site.url_main)})")
+            else:
+                if site.disabled:
+                    # Re-enable previously disabled site if DNS now resolves
+                    # (only if it was likely disabled due to DNS failure)
+                    pass
+        print(f"DNS check complete: {disabled_count} site(s) disabled, {len(failed)} domain(s) unresolvable.")
+
+    majestic_ranks = {}
+    if args.with_rank:
+        majestic_ranks = fetch_majestic_million()
+
+    for site in sites_subset:
+        if not args.with_rank:
+            break
+
+        if site.alexa_rank < sys.maxsize and args.empty_only:
+            continue
+        if args.exclude_engine_list and site.engine in args.exclude_engine_list:
+            continue
+
+        domain = get_base_domain(site.url_main)
+
+        if domain in majestic_ranks:
+            site.alexa_rank = majestic_ranks[domain]
+        else:
+            site.alexa_rank = sys.maxsize
+
+    # In memory matching complete, no threads to join
+    if args.with_rank:
+        print("Successfully updated ranks matching Majestic Million dataset.")
+
+    sites_full_list = [(s, int(s.alexa_rank)) for s in sites_subset]
+
+    sites_full_list.sort(reverse=False, key=lambda x: x[1])
+
+    while sites_full_list[0][1] == 0:
+        site = sites_full_list.pop(0)
+        sites_full_list.append(site)
+
+    for num, site_tuple in enumerate(sites_full_list):
+        site, rank = site_tuple
+        url_main = site.url_main
+        valid_rank = get_step_rank(rank)
+        all_tags = site.tags
+        all_tags.sort()
+        tags = ', ' + ', '.join(all_tags) if all_tags else ''
+        note = ''
+        if site.disabled:
+            note = ', search is disabled'
+
+        favicon = f"![](https://www.google.com/s2/favicons?domain={url_main})"
+        site_file.write(f'1. {favicon} [{site}]({url_main})*: top {valid_rank}{tags}*{note}\n')
+        db.update_site(site)
+
+    site_file.write(f'\nThe list was updated at ({datetime.now(timezone.utc).date()})\n')
+    db.save_to_file(args.base_file)
+
+    statistics_text = db.get_db_stats(is_markdown=True)
+    site_file.write('## Statistics\n\n')
+    site_file.write(statistics_text)
+
+    sites_md_written = write_sites_md_if_changed(site_file.getvalue(), "sites.md")
+    if not sites_md_written:
+        print("sites.md unchanged, skipping write")
+
+    # Regenerate db_meta.json to stay in sync with data.json — also a no-op
+    # if only the timestamp would change.
+    try:
+        meta_path = os.path.join(os.path.dirname(args.base_file), "db_meta.json")
+        meta, meta_written = write_meta_if_changed(
+            args.base_file,
+            meta_path,
+            min_version="0.5.0",
+            data_url="https://raw.githubusercontent.com/soxoj/maigret/main/maigret/resources/data.json",
+        )
+        if meta_written:
+            print(f"Updated {meta_path} ({meta['sites_count']} sites)")
+        else:
+            print(f"{meta_path} unchanged, skipping write")
+    except Exception as e:
+        print(f"Warning: could not regenerate db_meta.json: {e}")
+
+    print("Finished updating supported site listing!")
+
+
+if __name__ == '__main__':
+    main()