chore: import upstream snapshot with attribution

2026-07-13 12:12:13 +08:00
commit 0446c45d8e
898 changed files with 328024 additions and 0 deletions
@@ -0,0 +1,82 @@
+"""
+DomainMapper Demo — Comprehensive domain URL discovery
+
+Discovers all URLs under a domain using 8 sources:
+sitemap, Common Crawl, Wayback Machine, Certificate Transparency,
+path probing, robots.txt mining, RSS/Atom feeds, homepage link extraction.
+
+Usage:
+    python domain_mapper_demo.py [domain] [--source SOURCE] [--query QUERY]
+
+Examples:
+    python domain_mapper_demo.py superdesign.dev
+    python domain_mapper_demo.py example.com --source sitemap+crt+probe
+    python domain_mapper_demo.py docs.crawl4ai.com --query "extraction tutorial"
+"""
+
+import asyncio
+import argparse
+from collections import defaultdict
+from crawl4ai import DomainMapper, DomainMapperConfig
+
+
+async def main():
+    parser = argparse.ArgumentParser(description="DomainMapper Demo")
+    parser.add_argument("domain", help="Domain to scan (e.g., example.com)")
+    parser.add_argument("--source", default="sitemap+cc+crt+probe+robots+homepage",
+                        help="Discovery sources (default: sitemap+cc+crt+probe+robots+homepage)")
+    parser.add_argument("--query", default=None, help="BM25 relevance query")
+    parser.add_argument("--max-urls", type=int, default=-1, help="Max URLs to return")
+    parser.add_argument("--no-head", action="store_true", help="Skip head extraction")
+    args = parser.parse_args()
+
+    config = DomainMapperConfig(
+        source=args.source,
+        extract_head=not args.no_head,
+        query=args.query,
+        max_urls=args.max_urls,
+        verbose=True,
+        force=True,
+    )
+
+    async with DomainMapper() as mapper:
+        results = await mapper.scan(args.domain, config)
+
+    # Group by host
+    by_host = defaultdict(list)
+    for r in results:
+        by_host[r["host"]].append(r)
+
+    # Print results
+    print("\n" + "=" * 70)
+    print(f"RESULTS: {len(results)} URLs across {len(by_host)} hosts")
+    print("=" * 70)
+
+    for host in sorted(by_host.keys()):
+        urls = by_host[host]
+        print(f"\n  {host} ({len(urls)} URLs):")
+        for r in urls[:10]:
+            title = r.get("head_data", {}).get("title", "")
+            score = r.get("relevance_score")
+            line = f"    [{r['source']}] {r['url']}"
+            if title:
+                line += f"\n      Title: {title}"
+            if score is not None:
+                line += f"\n      Score: {score:.3f}"
+            print(line)
+        if len(urls) > 10:
+            print(f"    ... and {len(urls) - 10} more")
+
+    # Source breakdown
+    source_counts = defaultdict(int)
+    for r in results:
+        for s in r["source"].split("+"):
+            source_counts[s] += 1
+
+    print(f"\nSource breakdown:")
+    for source, count in sorted(source_counts.items(), key=lambda x: -x[1]):
+        print(f"  {source}: {count} URLs")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())