chore: import upstream snapshot with attribution

2026-07-13 12:12:13 +08:00
commit 0446c45d8e
898 changed files with 328024 additions and 0 deletions
@@ -0,0 +1,131 @@
+"""
+Test only the config matching logic without running crawler
+"""
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from crawl4ai.async_configs import CrawlerRunConfig, MatchMode
+
+def test_all_matching_scenarios():
+    print("Testing CrawlerRunConfig.is_match() method")
+    print("=" * 50)
+    
+    # Test 1: Single string pattern
+    print("\n1. Single string pattern (glob style)")
+    config = CrawlerRunConfig(
+        url_matcher="*.pdf",
+        # For example we can set this => scraping_strategy=PDFContentScrapingStrategy()
+    )
+    test_urls = [
+        ("https://example.com/file.pdf", True),
+        ("https://example.com/doc.PDF", False),  # Case sensitive
+        ("https://example.com/file.txt", False),
+        ("file.pdf", True),
+    ]
+    for url, expected in test_urls:
+        result = config.is_match(url)
+        status = "✓" if result == expected else "✗"
+        print(f"  {status} {url} -> {result}")
+    
+    # Test 2: List of patterns with OR
+    print("\n2. List of patterns with OR (default)")
+    config = CrawlerRunConfig(
+        url_matcher=["*/article/*", "*/blog/*", "*.html"],
+        match_mode=MatchMode.OR
+    )
+    test_urls = [
+        ("https://example.com/article/news", True),
+        ("https://example.com/blog/post", True),
+        ("https://example.com/page.html", True),
+        ("https://example.com/page.php", False),
+    ]
+    for url, expected in test_urls:
+        result = config.is_match(url)
+        status = "✓" if result == expected else "✗"
+        print(f"  {status} {url} -> {result}")
+    
+    # Test 3: Custom function
+    print("\n3. Custom function matcher")
+    config = CrawlerRunConfig(
+        url_matcher=lambda url: 'api' in url and (url.endswith('.json') or url.endswith('.xml'))
+    )
+    test_urls = [
+        ("https://api.example.com/data.json", True),
+        ("https://api.example.com/data.xml", True),
+        ("https://api.example.com/data.html", False),
+        ("https://example.com/data.json", False),  # No 'api'
+    ]
+    for url, expected in test_urls:
+        result = config.is_match(url)
+        status = "✓" if result == expected else "✗"
+        print(f"  {status} {url} -> {result}")
+    
+    # Test 4: Mixed list with AND
+    print("\n4. Mixed patterns and functions with AND")
+    config = CrawlerRunConfig(
+        url_matcher=[
+            "https://*",  # Must be HTTPS
+            lambda url: '.com' in url,  # Must have .com
+            lambda url: len(url) < 50  # Must be short
+        ],
+        match_mode=MatchMode.AND
+    )
+    test_urls = [
+        ("https://example.com/page", True),
+        ("http://example.com/page", False),  # Not HTTPS
+        ("https://example.org/page", False),  # No .com
+        ("https://example.com/" + "x" * 50, False),  # Too long
+    ]
+    for url, expected in test_urls:
+        result = config.is_match(url)
+        status = "✓" if result == expected else "✗"
+        print(f"  {status} {url} -> {result}")
+    
+    # Test 5: Complex real-world scenario
+    print("\n5. Complex pattern combinations")
+    config = CrawlerRunConfig(
+        url_matcher=[
+            "*/api/v[0-9]/*",  # API versioned endpoints
+            lambda url: 'graphql' in url,  # GraphQL endpoints
+            "*.json"  # JSON files
+        ],
+        match_mode=MatchMode.OR
+    )
+    test_urls = [
+        ("https://example.com/api/v1/users", True),
+        ("https://example.com/api/v2/posts", True),
+        ("https://example.com/graphql", True),
+        ("https://example.com/data.json", True),
+        ("https://example.com/api/users", False),  # No version
+    ]
+    for url, expected in test_urls:
+        result = config.is_match(url)
+        status = "✓" if result == expected else "✗"
+        print(f"  {status} {url} -> {result}")
+    
+    # Test 6: Edge cases
+    print("\n6. Edge cases")
+    
+    # No matcher
+    config = CrawlerRunConfig()
+    result = config.is_match("https://example.com")
+    print(f"  {'✓' if not result else '✗'} No matcher -> {result}")
+    
+    # Empty list
+    config = CrawlerRunConfig(url_matcher=[])
+    result = config.is_match("https://example.com")
+    print(f"  {'✓' if not result else '✗'} Empty list -> {result}")
+    
+    # None in list (should be skipped)
+    config = CrawlerRunConfig(url_matcher=["*.pdf", None, "*.doc"])
+    result = config.is_match("test.pdf")
+    print(f"  {'✓' if result else '✗'} List with None -> {result}")
+    
+    print("\n" + "=" * 50)
+    print("All matching tests completed!")
+
+if __name__ == "__main__":
+    test_all_matching_scenarios()